def test_errors_cumulative(self):
aggregator = ConsolidatingAggregator()
aggregator.track_percentiles = [50]
aggregator.prepare()
reader = self.get_fail_reader()
aggregator.add_underling(reader)
aggregator.shutdown()
aggregator.post_process()
cum_dict = aggregator.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_errors_cumulative(self):
aggregator = ConsolidatingAggregator()
aggregator.track_percentiles = [50]
aggregator.prepare()
reader = self.get_fail_reader()
aggregator.add_underling(reader)
aggregator.shutdown()
aggregator.post_process()
cum_dict = aggregator.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
self.obj.engine=EngineEmul()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
self.assertEqual(2, len(point[DataPoint.SUBRESULTS]))
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (self.obj.generalize_labels + 1) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit + 1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels + 1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
label_count = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(label_count * 2), count=label_count)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = label_count
self.obj.add_underling(reader)
last = None
for point in self.obj.datapoints(True):
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, label_count + 1) # didn't overflow
self.assertGreaterEqual(labels_count, label_count * 0.25) # at least a quarter-filled
self.assertEqual(1, len(last[DataPoint.SUBRESULTS]))
self.assertEqual(last, last[DataPoint.SUBRESULTS][0])
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors),
self.obj.max_error_count / 2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
if point[DataPoint.SUBRESULTS] == [point]:
del point[DataPoint.SUBRESULTS]
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (self.obj.generalize_labels + 1) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit + 1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels + 1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
LABEL_COUNT = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(LABEL_COUNT * 2), count=LABEL_COUNT)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = LABEL_COUNT
self.obj.add_underling(reader)
for point in self.obj.datapoints():
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, LABEL_COUNT + 1) # didn't overflow
self.assertGreaterEqual(labels_count, LABEL_COUNT * 0.25) # at least a quorter-filled
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors), self.obj.max_error_count / 2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
self.obj.engine = EngineEmul()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
self.assertEqual(2, len(point[DataPoint.SUBRESULTS]))
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_new_aggregator(self):
# aggregator's config
self.obj.extend_aggregation = True
reader = MockReader()
watcher = MockReader()
# executor/reporter prepare level
self.obj.add_underling(reader)
self.obj.add_listener(watcher)
# send rules to underlings
self.obj.startup()
reader.buffer_scale_idx = '100.0'
# data format: t_stamp, label, conc, r_time, con_time, latency, r_code, error, trname, byte_count
reader.data.append((1, "a", 1, 1, 1, 1, 200, None, '', 0))
reader.data.append((2, "b", 1, 2, 2, 2, 200, 'OK', '', 0))
reader.data.append((2, "b", 1, 3, 3, 3, 404, "Not Found", '', 0))
reader.data.append((2, "c", 1, 4, 4, 4, 200, None, '', 0))
reader.data.append((3, "d", 1, 5, 5, 5, 200, None, '', 0))
reader.data.append((4, "b", 1, 6, 6, 6, 200, None, '', 0))
# let's collect data to seconds and send something aggregated to watcher
self.obj.shutdown()
self.obj.post_process()
data_points = watcher.results[-1][DataPoint.CUMULATIVE]
self.assertEquals(7, len(data_points))
sample_labels = {'a-0', 'b-0', 'b-1', 'b-2', 'c-0', 'd-0', ''}
self.assertEquals(sample_labels, set(data_points.keys()))
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (self.obj.generalize_labels + 1) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit + 1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels + 1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
label_count = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(label_count * 2), count=label_count)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = label_count
self.obj.add_underling(reader)
last = None
for point in self.obj.datapoints(True):
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, label_count + 1) # didn't overflow
self.assertGreaterEqual(labels_count, label_count * 0.25) # at least a quarter-filled
self.assertEqual(1, len(last[DataPoint.SUBRESULTS]))
self.assertEqual(last, last[DataPoint.SUBRESULTS][0])
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors),
self.obj.max_error_count / 2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
if point[DataPoint.SUBRESULTS] == [point]:
del point[DataPoint.SUBRESULTS]
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
def test_ramp_up_exclude(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
self.obj.engine.config['settings']['ramp-up-exclude'] = True
self.obj.engine.config['execution'] = [
{'scenario': 'first', 'ramp-up': 50},
{'scenario': 'second', 'ramp-up': '1s'},
{'scenario': 'third'}
]
self.obj.engine.config['scenarios'] = BetterDict.from_dict({
'first': {'requests': [{'url': 'first'}]},
'second': {'requests': [{'url': 'second'}]},
'third': {'requests': [{'url': 'third'}]}})
reader = get_success_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
self.assertEquals(self.obj.cumulative, {})
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 9
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertLessEqual(len(cum_dict['']['errors']), self.obj.max_error_count)
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['rtimes-len'] = 42
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(42, kpiset.rtimes_len)
for kpiset in dp['current'].values():
self.assertNotEqual(42, kpiset.rtimes_len)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())