class TestMaster(ScenarioFixture, TestCase): maxDiff = None def setUp(self): # Set our test scenario differently from the default; must be BEFORE # the super call. self.scenario_dict = dict( name='Master Test Scenario - ablkei', sizes=[ dict(name='tiny', size_min=99, size_max=100), dict(name='small', size_min=1990, size_max=1990, crud_profile=[71, 9, 12, 8]), dict(name='medium', size_min=2990, size_max=3000), dict(name='unused', size_min=9876543, size_max=9876543), dict(name='large', size_min=399000, size_max=400000, crud_profile=[16, 61, 7, 16]), dict(name='huge', size_min=49900000, size_max=71499999)], initial_files=dict( tiny=300, small=400, medium=500, large=200, huge=70, ), operation_count=5000, # C R U D crud_profile=[5, 3, 1, 1], user_count=2, ) super(TestMaster, self).setUp() self.zmq_host = 'slick.queue.com' self.zmq_work_port = 7482 self.zmq_results_port = 18398 self.work_endpoint = 'tcp://%s:%d' % (self.zmq_host, self.zmq_work_port) self.results_endpoint = 'tcp://%s:%d' % (self.zmq_host, self.zmq_results_port) self.mock_context = flexmock() flexmock(zmq.Context).new_instances(self.mock_context).once self.mock_work_push = flexmock() self.mock_context.should_receive('socket').with_args( zmq.PUSH, ).and_return(self.mock_work_push).once self.mock_work_push.should_receive('bind').with_args( self.work_endpoint, ).once self.mock_results_pull = flexmock() self.mock_context.should_receive('socket').with_args( zmq.PULL, ).and_return(self.mock_results_pull).once self.mock_results_pull.should_receive('bind').with_args( self.results_endpoint, ).once self.master = Master(self.zmq_host, self.zmq_work_port, self.zmq_results_port, connect_timeout=3.14159, network_timeout=2.71828) self.result_index = 1 # for self.gen_result() self.stub_results = [ self.gen_result( 1, ssbench.CREATE_OBJECT, 'small', 100.0, 101.0, 103.0), self.gen_result( 1, ssbench.READ_OBJECT, 'tiny', 103.0, 103.1, 103.8), self.gen_result( 1, ssbench.CREATE_OBJECT, 'huge', 103.8, 105.0, 106.0), self.gen_result( 1, ssbench.UPDATE_OBJECT, 'large', 106.1, 106.3, 106.4), # # exceptions should be ignored dict(worker_id=2, type=ssbench.UPDATE_OBJECT, completed_at=39293.2, exception='wacky!', traceback='ugh'), self.gen_result( 2, ssbench.UPDATE_OBJECT, 'medium', 100.1, 100.9, 102.9), self.gen_result( 2, ssbench.DELETE_OBJECT, 'large', 102.9, 103.0, 103.3), self.gen_result( 2, ssbench.CREATE_OBJECT, 'tiny', 103.3, 103.4, 103.5), self.gen_result( 2, ssbench.READ_OBJECT, 'small', 103.5, 103.7, 104.0), # self.gen_result( 3, ssbench.READ_OBJECT, 'tiny', 100.1, 101.1, 101.9), # worker 3 took a while (observer lower concurrency in second 102 self.gen_result( 3, ssbench.DELETE_OBJECT, 'small', 103.1, 103.6, 103.9), self.gen_result( 3, ssbench.READ_OBJECT, 'medium', 103.9, 104.2, 104.3), self.gen_result( 3, ssbench.UPDATE_OBJECT, 'tiny', 104.3, 104.9, 104.999), ] def tearDown(self): super(TestMaster, self).tearDown() def gen_result(self, worker_id, op_type, size_str, start, first_byte, last_byte): self.result_index += 1 return { # There are other keys in a "result", but these are the only ones # used for the reporting. 'worker_id': worker_id, 'type': op_type, 'size_str': size_str, 'size': 989, 'first_byte_latency': first_byte - start, 'last_byte_latency': last_byte - start, 'trans_id': 'txID%03d' % self.result_index, 'completed_at': last_byte, } def test_format_bytes(self): input_expected = [ (0, ' 0 B'), (7, ' 7 B'), (17, ' 17 B'), (117, '117 B'), (999, '999 B'), (1000, ' 1 kB'), (1001, ' 1 kB'), (1100, ' 1 kB'), (1500, ' 2 kB'), (1999, ' 2 kB'), (2000, ' 2 kB'), (7500, ' 8 kB'), (7900, ' 8 kB'), (9999, ' 10 kB'), (10000, ' 10 kB'), (19999, ' 20 kB'), (20000, ' 20 kB'), (84999, ' 85 kB'), (85000, ' 85 kB'), (99999, '100 kB'), (100000, '100 kB'), (100001, '100 kB'), (700500, '701 kB'), (999999, ' 1 MB'), (1000000, ' 1 MB'), (1000001, ' 1 MB'), (4123456, ' 4 MB'), (4543210, ' 5 MB'), (99999999, '100 MB'), (100000000, '100 MB'), (9999999999, ' 10 GB'), (10000000000, ' 10 GB'), (99999999999, '100 GB'), (100000000000, '100 GB'), (234567890123, '235 GB'), ] for input_bytes, formatting in input_expected: self.assertEqual( formatting, self.master._format_bytes(input_bytes), '_format_bytes(%d) = %s instead of %s' % ( input_bytes, self.master._format_bytes(input_bytes), formatting)) def test_calculate_scenario_stats_aggregate(self): first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1, 0.2, 1, 0.5, 0.3, 0.6] last_byte_latency_all = [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2, 0.5, 1.8, 0.8, 0.4, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( worker_count=3, start=100.0, stop=106.4, req_count=12, avg_req_per_sec=round(12 / (106.4 - 100), 6), first_byte_latency=dict( min='%6.3f' % 0.1, max='%7.3f' % 1.2, avg='%7.3f' % stats.lmean(first_byte_latency_all), pctile='%7.3f' % 1.2, std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all), median='%7.3f' % stats.lmedianscore(first_byte_latency_all), ), last_byte_latency=dict( min='%6.3f' % 0.2, max='%7.3f' % 3.0, avg='%7.3f' % stats.lmean(last_byte_latency_all), pctile='%7.3f' % 3.0, std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all), median=' 0.749', # XXX why?? # median='%7.3f' % stats.lmedianscore(last_byte_latency_all), ), worst_first_byte_latency=(1.2, 'txID004'), worst_last_byte_latency=(3.0, 'txID002'), ), scen_stats['agg_stats']) def test_calculate_scenario_stats_aggregate_low_pctile(self): first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1, 0.2, 1, 0.5, 0.3, 0.6] last_byte_latency_all = [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2, 0.5, 1.8, 0.8, 0.4, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results, nth_pctile=20) self.assertDictEqual(dict( worker_count=3, start=100.0, stop=106.4, req_count=12, avg_req_per_sec=round(12 / (106.4 - 100), 6), first_byte_latency=dict( min='%6.3f' % 0.1, max='%7.3f' % 1.2, avg='%7.3f' % stats.lmean(first_byte_latency_all), pctile='%7.3f' % sorted(first_byte_latency_all)[2], std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all), median='%7.3f' % stats.lmedianscore(first_byte_latency_all), ), last_byte_latency=dict( min='%6.3f' % 0.2, max='%7.3f' % 3.0, avg='%7.3f' % stats.lmean(last_byte_latency_all), pctile='%7.3f' % sorted(last_byte_latency_all)[2], std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all), median=' 0.749', # XXX why?? # median='%7.3f' % stats.lmedianscore(last_byte_latency_all), ), worst_first_byte_latency=(1.2, 'txID004'), worst_last_byte_latency=(3.0, 'txID002'), ), scen_stats['agg_stats']) def test_calculate_scenario_stats_worker1(self): w1_first_byte_latency = [1.0, 0.1, 1.2, 0.2] w1_last_byte_latency = [3.0, 0.8, 2.2, 0.3] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.0, stop=106.4, req_count=4, avg_req_per_sec=round(4 / (106.4 - 100), 6), first_byte_latency=dict( min='%6.3f' % min(w1_first_byte_latency), max='%7.3f' % max(w1_first_byte_latency), avg='%7.3f' % stats.lmean(w1_first_byte_latency), pctile='%7.3f' % 1.2, std_dev='%7.3f' % stats.lsamplestdev(w1_first_byte_latency), median='%7.3f' % stats.lmedianscore(w1_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w1_last_byte_latency), max='%7.3f' % max(w1_last_byte_latency), avg='%7.3f' % stats.lmean(w1_last_byte_latency), pctile='%7.3f' % 3.0, std_dev='%7.3f' % stats.lsamplestdev(w1_last_byte_latency), median='%7.3f' % stats.lmedianscore(w1_last_byte_latency), ), worst_first_byte_latency=(float(max(w1_first_byte_latency)), 'txID004'), worst_last_byte_latency=( float(max(w1_last_byte_latency)), 'txID002'), ), scen_stats['worker_stats'][1]) def test_calculate_scenario_stats_worker2(self): w2_first_byte_latency = [0.8, 0.1, 0.1, 0.2] w2_last_byte_latency = [2.8, 0.4, 0.2, 0.5] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.0, req_count=4, avg_req_per_sec=round(4 / (104.0 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(w2_first_byte_latency), max='%7.3f' % max(w2_first_byte_latency), avg='%7.3f' % stats.lmean(w2_first_byte_latency), pctile='%7.3f' % 0.8, std_dev='%7.3f' % stats.lsamplestdev(w2_first_byte_latency), median='%7.3f' % stats.lmedianscore(w2_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w2_last_byte_latency), max='%7.3f' % max(w2_last_byte_latency), avg='%7.3f' % stats.lmean(w2_last_byte_latency), pctile='%7.3f' % 2.8, std_dev='%7.3f' % stats.lsamplestdev(w2_last_byte_latency), median='%7.3f' % stats.lmedianscore(w2_last_byte_latency), ), worst_first_byte_latency=(float(max(w2_first_byte_latency)), 'txID006'), worst_last_byte_latency=( float(max(w2_last_byte_latency)), 'txID006'), ), scen_stats['worker_stats'][2]) def test_calculate_scenario_stats_worker3(self): w3_first_byte_latency = [1, 0.5, 0.3, 0.6] w3_last_byte_latency = [1.8, 0.8, 0.4, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.999, req_count=4, avg_req_per_sec=round(4 / (104.999 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(w3_first_byte_latency), max='%7.3f' % max(w3_first_byte_latency), avg='%7.3f' % stats.lmean(w3_first_byte_latency), pctile='%7.3f' % 1.0, std_dev='%7.3f' % stats.lsamplestdev(w3_first_byte_latency), median='%7.3f' % stats.lmedianscore(w3_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w3_last_byte_latency), max='%7.3f' % max(w3_last_byte_latency), avg='%7.3f' % stats.lmean(w3_last_byte_latency), pctile='%7.3f' % 1.8, std_dev='%7.3f' % stats.lsamplestdev(w3_last_byte_latency), median='%7.3f' % stats.lmedianscore(w3_last_byte_latency), ), worst_first_byte_latency=( float(max(w3_first_byte_latency)), 'txID010'), worst_last_byte_latency=( float(max(w3_last_byte_latency)), 'txID010'), ), scen_stats['worker_stats'][3]) def test_calculate_scenario_stats_create(self): # Stats for Create c_first_byte_latency = [1, 1.2, 0.1] c_last_byte_latency = [3.0, 2.2, 0.2] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.0, stop=106.0, req_count=3, avg_req_per_sec=round(3 / (106 - 100.0), 6), first_byte_latency=dict( min='%6.3f' % min(c_first_byte_latency), max='%7.3f' % max(c_first_byte_latency), pctile='%7.3f' % max(c_first_byte_latency), avg='%7.3f' % stats.lmean(c_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(c_first_byte_latency), median='%7.3f' % stats.lmedianscore(c_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(c_last_byte_latency), max='%7.3f' % max(c_last_byte_latency), pctile='%7.3f' % max(c_last_byte_latency), avg='%7.3f' % stats.lmean(c_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(c_last_byte_latency), median='%7.3f' % stats.lmedianscore(c_last_byte_latency), ), worst_first_byte_latency=(max(c_first_byte_latency), 'txID004'), worst_last_byte_latency=(max(c_last_byte_latency), 'txID002'), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 5.0, 'first_byte_latency': {'avg': '%7.3f' % 0.1, 'max': '%7.3f' % 0.1, 'pctile': '%7.3f' % 0.1, 'median': '%7.3f' % 0.1, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'pctile': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.1, 'txID008'), 'worst_last_byte_latency': (0.2, 'txID008'), 'req_count': 1, 'start': 103.3, 'stop': 103.5}), ('small', {'avg_req_per_sec': 0.333333, 'first_byte_latency': {'avg': '%7.3f' % 1.0, 'max': '%7.3f' % 1.0, 'pctile': '%7.3f' % 1.0, 'median': '%7.3f' % 1.0, 'min': '%6.3f' % 1.0, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 3.0, 'max': '%7.3f' % 3.0, 'pctile': '%7.3f' % 3.0, 'median': '%7.3f' % 3.0, 'min': '%6.3f' % 3.0, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (1.0, 'txID002'), 'worst_last_byte_latency': (3.0, 'txID002'), 'req_count': 1, 'start': 100.0, 'stop': 103.0}), ('huge', {'avg_req_per_sec': 0.454545, 'first_byte_latency': {'avg': '%7.3f' % 1.2, 'max': '%7.3f' % 1.2, 'pctile': '%7.3f' % 1.2, 'median': '%7.3f' % 1.2, 'min': '%6.3f' % 1.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 2.2, 'max': '%7.3f' % 2.2, 'pctile': '%7.3f' % 2.2, 'median': '%7.3f' % 2.2, 'min': '%6.3f' % 2.2, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (1.2, 'txID004'), 'worst_last_byte_latency': (2.2, 'txID004'), 'req_count': 1, 'start': 103.8, 'stop': 106.0})]), ), scen_stats['op_stats'][ssbench.CREATE_OBJECT]) def test_calculate_scenario_stats_read(self): # Stats for Read r_first_byte_latency = [0.1, 0.2, 1.0, 0.3] r_last_byte_latency = [0.8, 0.5, 1.8, 0.4] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.3, req_count=4, avg_req_per_sec=round(4 / (104.3 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(r_first_byte_latency), max='%7.3f' % max(r_first_byte_latency), pctile='%7.3f' % max(r_first_byte_latency), avg='%7.3f' % stats.lmean(r_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(r_first_byte_latency), median='%7.3f' % stats.lmedianscore(r_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(r_last_byte_latency), max='%7.3f' % max(r_last_byte_latency), pctile='%7.3f' % max(r_last_byte_latency), avg='%7.3f' % stats.lmean(r_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(r_last_byte_latency), median='%7.3f' % stats.lmedianscore(r_last_byte_latency), ), worst_first_byte_latency=(max(r_first_byte_latency), 'txID010'), worst_last_byte_latency=(max(r_last_byte_latency), 'txID010'), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 0.540541, 'first_byte_latency': {'avg': '%7.3f' % 0.55, 'max': '%7.3f' % 1.0, 'pctile': '%7.3f' % 1.0, 'median': '%7.3f' % 0.55, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.45}, 'last_byte_latency': {'avg': '%7.3f' % 1.3, 'max': '%7.3f' % 1.8, 'pctile': '%7.3f' % 1.8, 'median': '%7.3f' % 1.3, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.5}, 'worst_first_byte_latency': (1.0, 'txID010'), 'worst_last_byte_latency': (1.8, 'txID010'), 'req_count': 2, 'start': 100.1, 'stop': 103.8}), ('small', {'avg_req_per_sec': 2.0, 'first_byte_latency': {'avg': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'pctile': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.5, 'max': '%7.3f' % 0.5, 'pctile': '%7.3f' % 0.5, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.2, 'txID009'), 'worst_last_byte_latency': (0.5, 'txID009'), 'req_count': 1, 'start': 103.5, 'stop': 104.0}), ('medium', {'avg_req_per_sec': 2.5, 'first_byte_latency': {'avg': '%7.3f' % 0.3, 'max': '%7.3f' % 0.3, 'pctile': '%7.3f' % 0.3, 'median': '%7.3f' % 0.3, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.4, 'max': '%7.3f' % 0.4, 'pctile': '%7.3f' % 0.4, 'median': '%7.3f' % 0.4, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.3, 'txID012'), 'worst_last_byte_latency': (0.4, 'txID012'), 'req_count': 1, 'start': 103.9, 'stop': 104.3})]), ), scen_stats['op_stats'][ssbench.READ_OBJECT]) def test_calculate_scenario_stats_update(self): u_first_byte_latency = [0.2, 0.8, 0.6] u_last_byte_latency = [0.3, 2.8, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=106.4, req_count=3, avg_req_per_sec=round(3 / (106.4 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(u_first_byte_latency), max='%7.3f' % max(u_first_byte_latency), avg='%7.3f' % stats.lmean(u_first_byte_latency), pctile='%7.3f' % 0.8, std_dev='%7.3f' % stats.lsamplestdev(u_first_byte_latency), median='%7.3f' % stats.lmedianscore(u_first_byte_latency), ), worst_first_byte_latency=(max(u_first_byte_latency), 'txID006'), last_byte_latency=dict( min='%6.3f' % min(u_last_byte_latency), max='%7.3f' % max(u_last_byte_latency), avg='%7.3f' % stats.lmean(u_last_byte_latency), pctile='%7.3f' % 2.8, std_dev='%7.3f' % stats.lsamplestdev(u_last_byte_latency), median='%7.3f' % stats.lmedianscore(u_last_byte_latency), ), worst_last_byte_latency=(max(u_last_byte_latency), 'txID006'), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 1.430615, 'first_byte_latency': {'avg': '%7.3f' % 0.6, 'pctile': '%7.3f' % 0.6, 'max': '%7.3f' % 0.6, 'median': '%7.3f' % 0.6, 'min': '%6.3f' % 0.6, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.6, 'txID013'), 'last_byte_latency': {'avg': '%7.3f' % 0.699, 'pctile': '%7.3f' % 0.699, 'max': '%7.3f' % 0.699, 'median': '%7.3f' % 0.699, 'min': '%6.3f' % 0.699, 'std_dev': '%7.3f' % 0.0}, 'worst_last_byte_latency': (0.699, 'txID013'), 'req_count': 1, 'start': 104.3, 'stop': 104.999}), ('medium', {'avg_req_per_sec': 0.357143, 'first_byte_latency': {'avg': '%7.3f' % 0.8, 'pctile': '%7.3f' % 0.8, 'max': '%7.3f' % 0.8, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.8, 'txID006'), 'last_byte_latency': {'avg': '%7.3f' % 2.8, 'pctile': '%7.3f' % 2.8, 'max': '%7.3f' % 2.8, 'median': '%7.3f' % 2.8, 'min': '%6.3f' % 2.8, 'std_dev': '%7.3f' % 0.0}, 'worst_last_byte_latency': (2.8, 'txID006'), 'req_count': 1, 'start': 100.1, 'stop': 102.9}), ('large', {'avg_req_per_sec': 3.333333, 'first_byte_latency': {'avg': '%7.3f' % 0.2, 'pctile': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.2, 'txID005'), 'last_byte_latency': {'avg': '%7.3f' % 0.3, 'pctile': '%7.3f' % 0.3, 'max': '%7.3f' % 0.3, 'median': '%7.3f' % 0.3, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.0}, 'worst_last_byte_latency': (0.3, 'txID005'), 'req_count': 1, 'start': 106.1, 'stop': 106.4})]), ), scen_stats['op_stats'][ssbench.UPDATE_OBJECT]) def test_calculate_scenario_stats_delete(self): d_first_byte_latency = [0.1, 0.5] d_last_byte_latency = [0.4, 0.8] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=102.9, stop=103.9, req_count=2, avg_req_per_sec=round(2 / (103.9 - 102.9), 6), first_byte_latency=dict( min='%6.3f' % min(d_first_byte_latency), max='%7.3f' % max(d_first_byte_latency), pctile='%7.3f' % max(d_first_byte_latency), avg='%7.3f' % stats.lmean(d_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(d_first_byte_latency), median='%7.3f' % stats.lmedianscore(d_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(d_last_byte_latency), max='%7.3f' % max(d_last_byte_latency), pctile='%7.3f' % max(d_last_byte_latency), avg='%7.3f' % stats.lmean(d_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(d_last_byte_latency), median='%7.3f' % stats.lmedianscore(d_last_byte_latency), ), worst_first_byte_latency=(max(d_first_byte_latency), 'txID011'), worst_last_byte_latency=(max(d_last_byte_latency), 'txID011'), size_stats=OrderedDict([ ('small', {'avg_req_per_sec': 1.25, 'first_byte_latency': {'avg': '%7.3f' % 0.5, 'max': '%7.3f' % 0.5, 'pctile': '%7.3f' % 0.5, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.8, 'max': '%7.3f' % 0.8, 'pctile': '%7.3f' % 0.8, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.5, 'txID011'), 'worst_last_byte_latency': (0.8, 'txID011'), 'req_count': 1, 'start': 103.1, 'stop': 103.9}), ('large', {'avg_req_per_sec': 2.5, 'first_byte_latency': {'avg': '%7.3f' % 0.1, 'max': '%7.3f' % 0.1, 'pctile': '%7.3f' % 0.1, 'median': '%7.3f' % 0.1, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.4, 'max': '%7.3f' % 0.4, 'pctile': '%7.3f' % 0.4, 'median': '%7.3f' % 0.4, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (0.1, 'txID007'), 'worst_last_byte_latency': (0.4, 'txID007'), 'req_count': 1, 'start': 102.9, 'stop': 103.3})]), ), scen_stats['op_stats'][ssbench.DELETE_OBJECT]) def test_calculate_scenario_size_stats(self): d_first_byte_latency = [0.1, 0.5] d_last_byte_latency = [0.4, 0.8] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(OrderedDict([ ('tiny', {'avg_req_per_sec': 0.816493, 'first_byte_latency': {'avg': '%7.3f' % 0.45, 'max': '%7.3f' % 1.0, 'pctile': '%7.3f' % 1.0, 'median': '%7.3f' % 0.35, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.377492}, 'last_byte_latency': {'avg': '%7.3f' % 0.87475, 'max': '%7.3f' % 1.8, 'pctile': '%7.3f' % 1.8, 'median': '%7.3f' % 0.7494, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.580485}, 'worst_first_byte_latency': (1.0, 'txID010'), 'worst_last_byte_latency': (1.8, 'txID010'), 'req_count': 4, 'start': 100.1, 'stop': 104.999}), ('small', {'avg_req_per_sec': 0.75, 'first_byte_latency': {'avg': '%7.3f' % 0.566667, 'max': '%7.3f' % 1.0, 'pctile': '%7.3f' % 1.0, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.329983}, 'last_byte_latency': {'avg': '%7.3f' % 1.433333, 'max': '%7.3f' % 3.0, 'pctile': '%7.3f' % 3.0, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 1.11455}, 'worst_first_byte_latency': (1.0, 'txID002'), 'worst_last_byte_latency': (3.0, 'txID002'), 'req_count': 3, 'start': 100.0, 'stop': 104.0}), ('medium', {'avg_req_per_sec': 0.47619, 'first_byte_latency': {'avg': '%7.3f' % 0.55, 'max': '%7.3f' % 0.8, 'pctile': '%7.3f' % 0.8, 'median': '%7.3f' % 0.55, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.25}, 'last_byte_latency': {'avg': '%7.3f' % 1.6, 'max': '%7.3f' % 2.8, 'pctile': '%7.3f' % 2.8, 'median': '%7.3f' % 1.6, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 1.2}, 'worst_first_byte_latency': (0.8, 'txID006'), 'worst_last_byte_latency': (2.8, 'txID006'), 'req_count': 2, 'start': 100.1, 'stop': 104.3}), ('large', {'avg_req_per_sec': 0.571429, 'first_byte_latency': {'avg': '%7.3f' % 0.15, 'max': '%7.3f' % 0.2, 'pctile': '%7.3f' % 0.2, 'median': '%7.3f' % 0.15, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.05}, 'last_byte_latency': {'avg': '%7.3f' % 0.35, 'max': '%7.3f' % 0.4, 'pctile': '%7.3f' % 0.4, 'median': '%7.3f' % 0.35, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.05}, 'worst_first_byte_latency': (0.2, 'txID005'), 'worst_last_byte_latency': (0.4, 'txID007'), 'req_count': 2, 'start': 102.9, 'stop': 106.4}), ('huge', {'avg_req_per_sec': 0.454545, 'first_byte_latency': {'avg': '%7.3f' % 1.2, 'max': '%7.3f' % 1.2, 'pctile': '%7.3f' % 1.2, 'median': '%7.3f' % 1.2, 'min': '%6.3f' % 1.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 2.2, 'max': '%7.3f' % 2.2, 'pctile': '%7.3f' % 2.2, 'median': '%7.3f' % 2.2, 'min': '%6.3f' % 2.2, 'std_dev': '%7.3f' % 0.0}, 'worst_first_byte_latency': (1.2, 'txID004'), 'worst_last_byte_latency': (2.2, 'txID004'), 'req_count': 1, 'start': 103.8, 'stop': 106.0})]), scen_stats['size_stats']) def test_calculate_scenario_stats_time_series(self): # Time series (reqs completed each second scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=101, start_time=99.19999999999999, stop=106, data=[1, 1, 5, 3, 0, 2], ), scen_stats['time_series']) def test_write_rps_histogram(self): # Write out time series data (requests-per-second histogram) to an # already open CSV file scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) test_csv_file = StringIO() self.master.write_rps_histogram(scen_stats, test_csv_file) test_csv_file.seek(0) reader = csv.reader(test_csv_file) self.assertListEqual([ ["Seconds Since Start", "Requests Completed"], ['1', '1'], ['2', '1'], ['3', '5'], ['4', '3'], ['5', '0'], ['6', '2'], ], list(reader)) def test_generate_scenario_report(self): # Time series (reqs completed each second scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results, nth_pctile=50) self.assertListEqual(u""" Master Test Scenario - ablkei Worker count: 3 Concurrency: 2 Ran 1970-01-01 00:01:39 UTC to 1970-01-01 00:01:46 UTC (7s) % Ops C R U D Size Range Size Name 20% % 50 30 10 10 99 B - 100 B tiny 27% % 71 9 12 8 2 kB small 34% % 50 30 10 10 3 kB - 3 kB medium 0% % 50 30 10 10 10 MB unused 14% % 16 61 7 16 399 kB - 400 kB large 5% % 50 30 10 10 50 MB - 71 MB huge --------------------------------------------------------------------- 51 29 10 10 CRUD weighted average TOTAL Count: 12 Average requests per second: 1.9 min max avg std_dev 50%-ile Worst latency TX ID First-byte latency: 0.100 - 1.200 0.508 ( 0.386) 0.400 (all obj sizes) txID004 Last-byte latency: 0.200 - 3.000 1.158 ( 0.970) 0.749 (all obj sizes) txID002 First-byte latency: 0.100 - 1.000 0.450 ( 0.377) 0.350 ( tiny objs) txID010 Last-byte latency: 0.200 - 1.800 0.875 ( 0.580) 0.749 ( tiny objs) txID010 First-byte latency: 0.200 - 1.000 0.567 ( 0.330) 0.500 ( small objs) txID002 Last-byte latency: 0.500 - 3.000 1.433 ( 1.115) 0.800 ( small objs) txID002 First-byte latency: 0.300 - 0.800 0.550 ( 0.250) 0.550 ( medium objs) txID006 Last-byte latency: 0.400 - 2.800 1.600 ( 1.200) 1.600 ( medium objs) txID006 First-byte latency: 0.100 - 0.200 0.150 ( 0.050) 0.150 ( large objs) txID005 Last-byte latency: 0.300 - 0.400 0.350 ( 0.050) 0.350 ( large objs) txID007 First-byte latency: 1.200 - 1.200 1.200 ( 0.000) 1.200 ( huge objs) txID004 Last-byte latency: 2.200 - 2.200 2.200 ( 0.000) 2.200 ( huge objs) txID004 CREATE Count: 3 Average requests per second: 0.5 min max avg std_dev 50%-ile Worst latency TX ID First-byte latency: 0.100 - 1.200 0.767 ( 0.478) 1.000 (all obj sizes) txID004 Last-byte latency: 0.200 - 3.000 1.800 ( 1.178) 2.200 (all obj sizes) txID002 First-byte latency: 0.100 - 0.100 0.100 ( 0.000) 0.100 ( tiny objs) txID008 Last-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 ( tiny objs) txID008 First-byte latency: 1.000 - 1.000 1.000 ( 0.000) 1.000 ( small objs) txID002 Last-byte latency: 3.000 - 3.000 3.000 ( 0.000) 3.000 ( small objs) txID002 First-byte latency: 1.200 - 1.200 1.200 ( 0.000) 1.200 ( huge objs) txID004 Last-byte latency: 2.200 - 2.200 2.200 ( 0.000) 2.200 ( huge objs) txID004 READ Count: 4 Average requests per second: 1.0 min max avg std_dev 50%-ile Worst latency TX ID First-byte latency: 0.100 - 1.000 0.400 ( 0.354) 0.250 (all obj sizes) txID010 Last-byte latency: 0.400 - 1.800 0.875 ( 0.554) 0.650 (all obj sizes) txID010 First-byte latency: 0.100 - 1.000 0.550 ( 0.450) 0.550 ( tiny objs) txID010 Last-byte latency: 0.800 - 1.800 1.300 ( 0.500) 1.300 ( tiny objs) txID010 First-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 ( small objs) txID009 Last-byte latency: 0.500 - 0.500 0.500 ( 0.000) 0.500 ( small objs) txID009 First-byte latency: 0.300 - 0.300 0.300 ( 0.000) 0.300 ( medium objs) txID012 Last-byte latency: 0.400 - 0.400 0.400 ( 0.000) 0.400 ( medium objs) txID012 UPDATE Count: 3 Average requests per second: 0.5 min max avg std_dev 50%-ile Worst latency TX ID First-byte latency: 0.200 - 0.800 0.533 ( 0.249) 0.600 (all obj sizes) txID006 Last-byte latency: 0.300 - 2.800 1.266 ( 1.097) 0.699 (all obj sizes) txID006 First-byte latency: 0.600 - 0.600 0.600 ( 0.000) 0.600 ( tiny objs) txID013 Last-byte latency: 0.699 - 0.699 0.699 ( 0.000) 0.699 ( tiny objs) txID013 First-byte latency: 0.800 - 0.800 0.800 ( 0.000) 0.800 ( medium objs) txID006 Last-byte latency: 2.800 - 2.800 2.800 ( 0.000) 2.800 ( medium objs) txID006 First-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 ( large objs) txID005 Last-byte latency: 0.300 - 0.300 0.300 ( 0.000) 0.300 ( large objs) txID005 DELETE Count: 2 Average requests per second: 2.0 min max avg std_dev 50%-ile Worst latency TX ID First-byte latency: 0.100 - 0.500 0.300 ( 0.200) 0.300 (all obj sizes) txID011 Last-byte latency: 0.400 - 0.800 0.600 ( 0.200) 0.600 (all obj sizes) txID011 First-byte latency: 0.500 - 0.500 0.500 ( 0.000) 0.500 ( small objs) txID011 Last-byte latency: 0.800 - 0.800 0.800 ( 0.000) 0.800 ( small objs) txID011 First-byte latency: 0.100 - 0.100 0.100 ( 0.000) 0.100 ( large objs) txID007 Last-byte latency: 0.400 - 0.400 0.400 ( 0.000) 0.400 ( large objs) txID007 """.split('\n'), self.master.generate_scenario_report(self.scenario, scen_stats).split('\n'))
class TestMaster(ScenarioFixture, TestCase): maxDiff = None def setUp(self): # Set our test scenario differently from the default; must be BEFORE # the super call. self.scenario_dict = dict( name='Master Test Scenario - ablkei', sizes=[ dict(name='tiny', size_min=99, size_max=100), dict(name='small', size_min=199, size_max=200), dict(name='medium', size_min=299, size_max=300), dict(name='large', size_min=399, size_max=400), dict(name='huge', size_min=499, size_max=500)], initial_files=dict( tiny=300, small=300, medium=300, large=100, huge=70, ), operation_count=5000, # C R U D crud_profile=[5, 3, 1, 1], user_count=2, ) super(TestMaster, self).setUp() self.stub_queue = flexmock() self.stub_queue.should_receive('watch').with_args(ssbench.STATS_TUBE).once self.stub_queue.should_receive('ignore').with_args(ssbench.DEFAULT_TUBE).once self.master = Master(self.stub_queue) self.result_index = 1 # for self.gen_result() self.stub_results = [ self.gen_result(1, ssbench.CREATE_OBJECT, 'small', 100.0, 101.0, 103.0), self.gen_result(1, ssbench.READ_OBJECT, 'tiny', 103.0, 103.1, 103.8), self.gen_result(1, ssbench.CREATE_OBJECT, 'huge', 103.8, 105.0, 106.0), self.gen_result(1, ssbench.UPDATE_OBJECT, 'large', 106.1, 106.3, 106.4), # # exceptions should be ignored dict(worker_id=2, type=ssbench.UPDATE_OBJECT, completed_at=39293.2, exception='wacky!'), self.gen_result(2, ssbench.UPDATE_OBJECT, 'medium', 100.1, 100.9, 102.9), self.gen_result(2, ssbench.DELETE_OBJECT, 'large', 102.9, 103.0, 103.3), self.gen_result(2, ssbench.CREATE_OBJECT, 'tiny', 103.3, 103.4, 103.5), self.gen_result(2, ssbench.READ_OBJECT, 'small', 103.5, 103.7, 104.0), # self.gen_result(3, ssbench.READ_OBJECT, 'tiny', 100.1, 101.1, 101.9), # worker 3 took a while (observer lower concurrency in second 102 self.gen_result(3, ssbench.DELETE_OBJECT, 'small', 103.1, 103.6, 103.9), self.gen_result(3, ssbench.READ_OBJECT, 'medium', 103.9, 104.2, 104.3), self.gen_result(3, ssbench.UPDATE_OBJECT, 'tiny', 104.3, 104.9, 104.999), ] def tearDown(self): super(TestMaster, self).tearDown() def gen_result(self, worker_id, op_type, size_str, start, first_byte, last_byte): self.result_index += 1 return { # There are other keys in a "result", but these are the only ones # used for the reporting. 'worker_id': worker_id, 'type': op_type, 'size_str': size_str, 'size': 989, 'first_byte_latency': first_byte - start, 'last_byte_latency': last_byte - start, 'completed_at': last_byte, } def test_calculate_scenario_stats_aggregate(self): first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1, 0.2, 1, 0.5, 0.3, 0.6] last_byte_latency_all = [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2, 0.5, 1.8, 0.8, 0.4, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( worker_count=3, start=100.0, stop=106.4, req_count=12, avg_req_per_sec=round(12 / (106.4 - 100), 6), first_byte_latency=dict( min='%6.3f' % 0.1, max='%7.3f' % 1.2, avg='%7.3f' % stats.lmean(first_byte_latency_all), std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all), median='%7.3f' % stats.lmedianscore(first_byte_latency_all), ), last_byte_latency=dict( min='%6.3f' % 0.2, max='%7.3f' % 3.0, avg='%7.3f' % stats.lmean(last_byte_latency_all), std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all), median=' 0.749', # XXX why?? #median='%7.3f' % stats.lmedianscore(last_byte_latency_all), ), ), scen_stats['agg_stats']) def test_calculate_scenario_stats_worker1(self): w1_first_byte_latency = [1.0, 0.1, 1.2, 0.2] w1_last_byte_latency = [3.0, 0.8, 2.2, 0.3] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.0, stop=106.4, req_count=4, avg_req_per_sec=round(4 / (106.4 - 100), 6), first_byte_latency=dict( min='%6.3f' % min(w1_first_byte_latency), max='%7.3f' % max(w1_first_byte_latency), avg='%7.3f' % stats.lmean(w1_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w1_first_byte_latency), median='%7.3f' % stats.lmedianscore(w1_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w1_last_byte_latency), max='%7.3f' % max(w1_last_byte_latency), avg='%7.3f' % stats.lmean(w1_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w1_last_byte_latency), median='%7.3f' % stats.lmedianscore(w1_last_byte_latency), ), ), scen_stats['worker_stats'][1]) def test_calculate_scenario_stats_worker2(self): w2_first_byte_latency = [0.8, 0.1, 0.1, 0.2] w2_last_byte_latency = [2.8, 0.4, 0.2, 0.5] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.0, req_count=4, avg_req_per_sec=round(4 / (104.0 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(w2_first_byte_latency), max='%7.3f' % max(w2_first_byte_latency), avg='%7.3f' % stats.lmean(w2_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w2_first_byte_latency), median='%7.3f' % stats.lmedianscore(w2_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w2_last_byte_latency), max='%7.3f' % max(w2_last_byte_latency), avg='%7.3f' % stats.lmean(w2_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w2_last_byte_latency), median='%7.3f' % stats.lmedianscore(w2_last_byte_latency), ), ), scen_stats['worker_stats'][2]) def test_calculate_scenario_stats_worker3(self): w3_first_byte_latency = [1, 0.5, 0.3, 0.6] w3_last_byte_latency = [1.8, 0.8, 0.4, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.999, req_count=4, avg_req_per_sec=round(4 / (104.999 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(w3_first_byte_latency), max='%7.3f' % max(w3_first_byte_latency), avg='%7.3f' % stats.lmean(w3_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w3_first_byte_latency), median='%7.3f' % stats.lmedianscore(w3_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(w3_last_byte_latency), max='%7.3f' % max(w3_last_byte_latency), avg='%7.3f' % stats.lmean(w3_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(w3_last_byte_latency), median='%7.3f' % stats.lmedianscore(w3_last_byte_latency), ), ), scen_stats['worker_stats'][3]) def test_calculate_scenario_stats_create(self): # Stats for Create c_first_byte_latency = [1, 1.2, 0.1] c_last_byte_latency = [3.0, 2.2, 0.2] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.0, stop=106.0, req_count=3, avg_req_per_sec=round(3 / (106 - 100.0), 6), first_byte_latency=dict( min='%6.3f' % min(c_first_byte_latency), max='%7.3f' % max(c_first_byte_latency), avg='%7.3f' % stats.lmean(c_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(c_first_byte_latency), median='%7.3f' % stats.lmedianscore(c_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(c_last_byte_latency), max='%7.3f' % max(c_last_byte_latency), avg='%7.3f' % stats.lmean(c_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(c_last_byte_latency), median='%7.3f' % stats.lmedianscore(c_last_byte_latency), ), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 5.0, 'first_byte_latency': {'avg': '%7.3f' % 0.1, 'max': '%7.3f' % 0.1, 'median': '%7.3f' % 0.1, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.3, 'stop': 103.5}), ('small', {'avg_req_per_sec': 0.333333, 'first_byte_latency': {'avg': '%7.3f' % 1.0, 'max': '%7.3f' % 1.0, 'median': '%7.3f' % 1.0, 'min': '%6.3f' % 1.0, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 3.0, 'max': '%7.3f' % 3.0, 'median': '%7.3f' % 3.0, 'min': '%6.3f' % 3.0, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 100.0, 'stop': 103.0}), ('huge', {'avg_req_per_sec': 0.454545, 'first_byte_latency': {'avg': '%7.3f' % 1.2, 'max': '%7.3f' % 1.2, 'median': '%7.3f' % 1.2, 'min': '%6.3f' % 1.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 2.2, 'max': '%7.3f' % 2.2, 'median': '%7.3f' % 2.2, 'min': '%6.3f' % 2.2, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.8, 'stop': 106.0})]), ), scen_stats['op_stats'][ssbench.CREATE_OBJECT]) def test_calculate_scenario_stats_read(self): # Stats for Read r_first_byte_latency = [0.1, 0.2, 1.0, 0.3] r_last_byte_latency = [0.8, 0.5, 1.8, 0.4] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=104.3, req_count=4, avg_req_per_sec=round(4 / (104.3 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(r_first_byte_latency), max='%7.3f' % max(r_first_byte_latency), avg='%7.3f' % stats.lmean(r_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(r_first_byte_latency), median='%7.3f' % stats.lmedianscore(r_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(r_last_byte_latency), max='%7.3f' % max(r_last_byte_latency), avg='%7.3f' % stats.lmean(r_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(r_last_byte_latency), median='%7.3f' % stats.lmedianscore(r_last_byte_latency), ), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 0.540541, 'first_byte_latency': {'avg': '%7.3f' % 0.55, 'max': '%7.3f' % 1.0, 'median': '%7.3f' % 0.55, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.45}, 'last_byte_latency': {'avg': '%7.3f' % 1.3, 'max': '%7.3f' % 1.8, 'median': '%7.3f' % 1.3, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.5}, 'req_count': 2, 'start': 100.1, 'stop': 103.8}), ('small', {'avg_req_per_sec': 2.0, 'first_byte_latency': {'avg': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.5, 'max': '%7.3f' % 0.5, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.5, 'stop': 104.0}), ('medium', {'avg_req_per_sec': 2.5, 'first_byte_latency': {'avg': '%7.3f' % 0.3, 'max': '%7.3f' % 0.3, 'median': '%7.3f' % 0.3, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.4, 'max': '%7.3f' % 0.4, 'median': '%7.3f' % 0.4, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.9, 'stop': 104.3})]), ), scen_stats['op_stats'][ssbench.READ_OBJECT]) def test_calculate_scenario_stats_update(self): u_first_byte_latency = [0.2, 0.8, 0.6] u_last_byte_latency = [0.3, 2.8, 0.699] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=100.1, stop=106.4, req_count=3, avg_req_per_sec=round(3 / (106.4 - 100.1), 6), first_byte_latency=dict( min='%6.3f' % min(u_first_byte_latency), max='%7.3f' % max(u_first_byte_latency), avg='%7.3f' % stats.lmean(u_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(u_first_byte_latency), median='%7.3f' % stats.lmedianscore(u_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(u_last_byte_latency), max='%7.3f' % max(u_last_byte_latency), avg='%7.3f' % stats.lmean(u_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(u_last_byte_latency), median='%7.3f' % stats.lmedianscore(u_last_byte_latency), ), size_stats=OrderedDict([ ('tiny', {'avg_req_per_sec': 1.430615, 'first_byte_latency': {'avg': '%7.3f' % 0.6, 'max': '%7.3f' % 0.6, 'median': '%7.3f' % 0.6, 'min': '%6.3f' % 0.6, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.699, 'max': '%7.3f' % 0.699, 'median': '%7.3f' % 0.699, 'min': '%6.3f' % 0.699, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 104.3, 'stop': 104.999}), ('medium', {'avg_req_per_sec': 0.357143, 'first_byte_latency': {'avg': '%7.3f' % 0.8, 'max': '%7.3f' % 0.8, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 2.8, 'max': '%7.3f' % 2.8, 'median': '%7.3f' % 2.8, 'min': '%6.3f' % 2.8, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 100.1, 'stop': 102.9}), ('large', {'avg_req_per_sec': 3.333333, 'first_byte_latency': {'avg': '%7.3f' % 0.2, 'max': '%7.3f' % 0.2, 'median': '%7.3f' % 0.2, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.3, 'max': '%7.3f' % 0.3, 'median': '%7.3f' % 0.3, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 106.1, 'stop': 106.4})]), ), scen_stats['op_stats'][ssbench.UPDATE_OBJECT]) def test_calculate_scenario_stats_delete(self): d_first_byte_latency = [0.1, 0.5] d_last_byte_latency = [0.4, 0.8] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=102.9, stop=103.9, req_count=2, avg_req_per_sec=round(2 / (103.9 - 102.9), 6), first_byte_latency=dict( min='%6.3f' % min(d_first_byte_latency), max='%7.3f' % max(d_first_byte_latency), avg='%7.3f' % stats.lmean(d_first_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(d_first_byte_latency), median='%7.3f' % stats.lmedianscore(d_first_byte_latency), ), last_byte_latency=dict( min='%6.3f' % min(d_last_byte_latency), max='%7.3f' % max(d_last_byte_latency), avg='%7.3f' % stats.lmean(d_last_byte_latency), std_dev='%7.3f' % stats.lsamplestdev(d_last_byte_latency), median='%7.3f' % stats.lmedianscore(d_last_byte_latency), ), size_stats=OrderedDict([ ('small', {'avg_req_per_sec': 1.25, 'first_byte_latency': {'avg': '%7.3f' % 0.5, 'max': '%7.3f' % 0.5, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.8, 'max': '%7.3f' % 0.8, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.8, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.1, 'stop': 103.9}), ('large', {'avg_req_per_sec': 2.5, 'first_byte_latency': {'avg': '%7.3f' % 0.1, 'max': '%7.3f' % 0.1, 'median': '%7.3f' % 0.1, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 0.4, 'max': '%7.3f' % 0.4, 'median': '%7.3f' % 0.4, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 102.9, 'stop': 103.3})]), ), scen_stats['op_stats'][ssbench.DELETE_OBJECT]) def test_calculate_scenario_size_stats(self): d_first_byte_latency = [0.1, 0.5] d_last_byte_latency = [0.4, 0.8] scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(OrderedDict([ ('tiny', {'avg_req_per_sec': 0.816493, 'first_byte_latency': {'avg': '%7.3f' % 0.45, 'max': '%7.3f' % 1.0, 'median': '%7.3f' % 0.35, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.377492}, 'last_byte_latency': {'avg': '%7.3f' % 0.87475, 'max': '%7.3f' % 1.8, 'median': '%7.3f' % 0.7494, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.580485}, 'req_count': 4, 'start': 100.1, 'stop': 104.999}), ('small', {'avg_req_per_sec': 0.75, 'first_byte_latency': {'avg': '%7.3f' % 0.566667, 'max': '%7.3f' % 1.0, 'median': '%7.3f' % 0.5, 'min': '%6.3f' % 0.2, 'std_dev': '%7.3f' % 0.329983}, 'last_byte_latency': {'avg': '%7.3f' % 1.433333, 'max': '%7.3f' % 3.0, 'median': '%7.3f' % 0.8, 'min': '%6.3f' % 0.5, 'std_dev': '%7.3f' % 1.11455}, 'req_count': 3, 'start': 100.0, 'stop': 104.0}), ('medium', {'avg_req_per_sec': 0.47619, 'first_byte_latency': {'avg': '%7.3f' % 0.55, 'max': '%7.3f' % 0.8, 'median': '%7.3f' % 0.55, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.25}, 'last_byte_latency': {'avg': '%7.3f' % 1.6, 'max': '%7.3f' % 2.8, 'median': '%7.3f' % 1.6, 'min': '%6.3f' % 0.4, 'std_dev': '%7.3f' % 1.2}, 'req_count': 2, 'start': 100.1, 'stop': 104.3}), ('large', {'avg_req_per_sec': 0.571429, 'first_byte_latency': {'avg': '%7.3f' % 0.15, 'max': '%7.3f' % 0.2, 'median': '%7.3f' % 0.15, 'min': '%6.3f' % 0.1, 'std_dev': '%7.3f' % 0.05}, 'last_byte_latency': {'avg': '%7.3f' % 0.35, 'max': '%7.3f' % 0.4, 'median': '%7.3f' % 0.35, 'min': '%6.3f' % 0.3, 'std_dev': '%7.3f' % 0.05}, 'req_count': 2, 'start': 102.9, 'stop': 106.4}), ('huge', {'avg_req_per_sec': 0.454545, 'first_byte_latency': {'avg': '%7.3f' % 1.2, 'max': '%7.3f' % 1.2, 'median': '%7.3f' % 1.2, 'min': '%6.3f' % 1.2, 'std_dev': '%7.3f' % 0.0}, 'last_byte_latency': {'avg': '%7.3f' % 2.2, 'max': '%7.3f' % 2.2, 'median': '%7.3f' % 2.2, 'min': '%6.3f' % 2.2, 'std_dev': '%7.3f' % 0.0}, 'req_count': 1, 'start': 103.8, 'stop': 106.0})]), scen_stats['size_stats']) def test_calculate_scenario_stats_time_series(self): # Time series (reqs completed each second scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertDictEqual(dict( start=101, data=[1, 1, 5, 3, 0, 2], ), scen_stats['time_series']) def test_write_rps_histogram(self): # Write out time series data (requests-per-second histogram) to an # already open CSV file scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) test_csv_file = StringIO() self.master.write_rps_histogram(scen_stats, test_csv_file) test_csv_file.seek(0) reader = csv.reader(test_csv_file) self.assertListEqual([ ["Seconds Since Start", "Requests Completed"], ['1', '1'], ['2', '1'], ['3', '5'], ['4', '3'], ['5', '0'], ['6', '2'], ], list(reader)) def test_generate_scenario_report(self): # Time series (reqs completed each second scen_stats = self.master.calculate_scenario_stats(self.scenario, self.stub_results) self.assertListEqual(u""" Master Test Scenario - ablkei C R U D Worker count: 3 Concurrency: 2 % 50 30 10 10 TOTAL Count: 12 Average requests per second: 1.9 min max avg std_dev median First-byte latency: 0.100 - 1.200 0.508 ( 0.386) 0.400 (all obj sizes) Last-byte latency: 0.200 - 3.000 1.158 ( 0.970) 0.749 (all obj sizes) First-byte latency: 0.100 - 1.000 0.450 ( 0.377) 0.350 (tiny objs) Last-byte latency: 0.200 - 1.800 0.875 ( 0.580) 0.749 (tiny objs) First-byte latency: 0.200 - 1.000 0.567 ( 0.330) 0.500 (small objs) Last-byte latency: 0.500 - 3.000 1.433 ( 1.115) 0.800 (small objs) First-byte latency: 0.300 - 0.800 0.550 ( 0.250) 0.550 (medium objs) Last-byte latency: 0.400 - 2.800 1.600 ( 1.200) 1.600 (medium objs) First-byte latency: 0.100 - 0.200 0.150 ( 0.050) 0.150 (large objs) Last-byte latency: 0.300 - 0.400 0.350 ( 0.050) 0.350 (large objs) First-byte latency: 1.200 - 1.200 1.200 ( 0.000) 1.200 (huge objs) Last-byte latency: 2.200 - 2.200 2.200 ( 0.000) 2.200 (huge objs) CREATE Count: 3 Average requests per second: 0.5 min max avg std_dev median First-byte latency: 0.100 - 1.200 0.767 ( 0.478) 1.000 (all obj sizes) Last-byte latency: 0.200 - 3.000 1.800 ( 1.178) 2.200 (all obj sizes) First-byte latency: 0.100 - 0.100 0.100 ( 0.000) 0.100 (tiny objs) Last-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 (tiny objs) First-byte latency: 1.000 - 1.000 1.000 ( 0.000) 1.000 (small objs) Last-byte latency: 3.000 - 3.000 3.000 ( 0.000) 3.000 (small objs) First-byte latency: 1.200 - 1.200 1.200 ( 0.000) 1.200 (huge objs) Last-byte latency: 2.200 - 2.200 2.200 ( 0.000) 2.200 (huge objs) READ Count: 4 Average requests per second: 1.0 min max avg std_dev median First-byte latency: 0.100 - 1.000 0.400 ( 0.354) 0.250 (all obj sizes) Last-byte latency: 0.400 - 1.800 0.875 ( 0.554) 0.650 (all obj sizes) First-byte latency: 0.100 - 1.000 0.550 ( 0.450) 0.550 (tiny objs) Last-byte latency: 0.800 - 1.800 1.300 ( 0.500) 1.300 (tiny objs) First-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 (small objs) Last-byte latency: 0.500 - 0.500 0.500 ( 0.000) 0.500 (small objs) First-byte latency: 0.300 - 0.300 0.300 ( 0.000) 0.300 (medium objs) Last-byte latency: 0.400 - 0.400 0.400 ( 0.000) 0.400 (medium objs) UPDATE Count: 3 Average requests per second: 0.5 min max avg std_dev median First-byte latency: 0.200 - 0.800 0.533 ( 0.249) 0.600 (all obj sizes) Last-byte latency: 0.300 - 2.800 1.266 ( 1.097) 0.699 (all obj sizes) First-byte latency: 0.600 - 0.600 0.600 ( 0.000) 0.600 (tiny objs) Last-byte latency: 0.699 - 0.699 0.699 ( 0.000) 0.699 (tiny objs) First-byte latency: 0.800 - 0.800 0.800 ( 0.000) 0.800 (medium objs) Last-byte latency: 2.800 - 2.800 2.800 ( 0.000) 2.800 (medium objs) First-byte latency: 0.200 - 0.200 0.200 ( 0.000) 0.200 (large objs) Last-byte latency: 0.300 - 0.300 0.300 ( 0.000) 0.300 (large objs) DELETE Count: 2 Average requests per second: 2.0 min max avg std_dev median First-byte latency: 0.100 - 0.500 0.300 ( 0.200) 0.300 (all obj sizes) Last-byte latency: 0.400 - 0.800 0.600 ( 0.200) 0.600 (all obj sizes) First-byte latency: 0.500 - 0.500 0.500 ( 0.000) 0.500 (small objs) Last-byte latency: 0.800 - 0.800 0.800 ( 0.000) 0.800 (small objs) First-byte latency: 0.100 - 0.100 0.100 ( 0.000) 0.100 (large objs) Last-byte latency: 0.400 - 0.400 0.400 ( 0.000) 0.400 (large objs) """.split('\n'), self.master.generate_scenario_report(self.scenario, scen_stats).split('\n'))