def test_set_rtimes_len(self):
obj = ConsolidatingAggregator()
obj.settings['rtimes-len'] = 42
obj.prepare()
reader = self.get_fail_reader()
obj.add_underling(reader)
listener = MockListener()
obj.add_listener(listener)
obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(42, kpiset.rtimes_len)
for kpiset in dp['current'].values():
self.assertNotEqual(42, kpiset.rtimes_len)
def test_datapoint_to_json(self):
obj = ConsolidatingAggregator()
obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
obj.prepare()
obj.add_underling(self.get_success_reader())
for point in obj.datapoints():
obj.log.info(to_json(point))
def test_errors_cumulative(self):
aggregator = ConsolidatingAggregator()
aggregator.track_percentiles = [50]
aggregator.prepare()
reader = self.get_fail_reader()
aggregator.add_underling(reader)
aggregator.shutdown()
aggregator.post_process()
cum_dict = aggregator.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_two_executions(self):
# check consolidator
obj = ConsolidatingAggregator()
obj.prepare()
obj.track_percentiles = [0, 50, 100]
underling1 = self.get_reader()
underling2 = self.get_reader()
obj.add_underling(underling1)
obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in obj.datapoints():
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
self.assertEqual(2, len(point[DataPoint.SUBRESULTS]))
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (
self.obj.generalize_labels + 1
) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels /
2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit +
1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels /
2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels +
1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
label_count = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(
label_count * 2),
count=label_count)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = label_count
self.obj.add_underling(reader)
last = None
for point in self.obj.datapoints():
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, label_count + 1) # didn't overflow
self.assertGreaterEqual(labels_count, label_count *
0.25) # at least a quarter-filled
self.assertEqual(1, len(last[DataPoint.SUBRESULTS]))
self.assertEqual(last, last[DataPoint.SUBRESULTS][0])
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors),
self.obj.max_error_count /
2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
if point[DataPoint.SUBRESULTS] == [point]:
del point[DataPoint.SUBRESULTS]
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append(
(7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append(
(6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported",
self.log_recorder.warn_buff.getvalue())
def setUp(self):
super(TestSiegeExecutor, self).setUp()
self.obj.engine.aggregator = ConsolidatingAggregator()
self.obj.settings.merge({"path": TOOL_PATH})
def test_simple(self):
obj = PBenchExecutor()
obj.engine = EngineEmul()
obj.engine.aggregator = ConsolidatingAggregator()
obj.engine.aggregator.add_listener(DataPointLogger())
obj.engine.config.merge({"provisioning": "test"})
if os.path.exists("/home/undera/Sources/phantom"
): # FIXME: not good, get rid of it
obj.settings.merge({
"path":
"/home/undera/Sources/phantom/bin/phantom",
"modules-path":
"/home/undera/Sources/phantom/lib/phantom",
})
else:
obj.settings.merge({
"path":
os.path.join(os.path.dirname(__file__), '..',
"phantom.sh"),
})
obj.execution.merge({
"log-responses": "proto_error",
# "iterations": 5000000,
"concurrency": 10,
"throughput": 1000,
"ramp-up": "1m",
# "steps": 5,
"hold-for": "15",
"scenario": {
"timeout":
1,
"default-address":
"http://localhost:33",
"headers": {
"Connection": "close"
},
"requests": [
# "/",
{
"url": "/api",
"method": "POST",
"headers": {
"Content-Length": 0
},
"body": {
"param": "value"
}
}
]
}
})
obj.engine.aggregator.prepare()
obj.prepare()
obj.engine.aggregator.startup()
obj.startup()
while not obj.check():
logging.debug("Running...")
obj.engine.aggregator.check()
time.sleep(1)
obj.shutdown()
obj.engine.aggregator.shutdown()
obj.post_process()
obj.engine.aggregator.post_process()
def configure(self, config):
super(TestExternalResultsLoader, self).configure(config)
self.results_listener = MockReader()
self.obj.engine.aggregator = ConsolidatingAggregator()
self.obj.engine.aggregator.engine = self.obj.engine
self.obj.engine.aggregator.add_listener(self.results_listener)
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
self.obj.engine = EngineEmul()
def test_extend_datapoints(self):
# check reported data format conversion for test state filtering on BM side
def get_mock(origin_func, store):
# generate replacement for BlazemeterUploader._dpoint_serializer.get_kpi_body
def mock_get_kpi_body(data, isfinal):
store.append(data) # save received data for verifying
return origin_func(
data, isfinal) # call original get_kpi_body as well
return mock_get_kpi_body
mock = BZMock()
mock.mock_get.update({
'1': {
"result": []
},
'https://a.blazemeter.com/api/v4/tests?projectId=1&name=Taurus+Test':
{
"result": []
},
'3': {
"result": []
},
})
mock.mock_post.update({
'https://a.blazemeter.com/api/v4/projects': {
"result": {
"id": 1
}
},
'https://a.blazemeter.com/api/v4/tests': {
"result": {
'id': 1
}
},
'https://a.blazemeter.com/api/v4/tests/1/start-external': {
"result": {
'session': {
'id': 1,
'userId': 1,
'testId': 1
},
'master': {
'id': 1,
'userId': 1
},
'signature': 'sign'
}
},
'https://data.blazemeter.com/submit.php?session_id=1&signature=sign&test_id=1&user_id=1&pq=0&target=labels_bulk&update=1':
[
{},
{
"result": {
'session': {
"statusCode": 140,
'status': 'ENDED'
}
}
},
{},
],
'https://data.blazemeter.com/api/v4/image/1/files?signature=sign':
[
IOError("monitoring push expected fail"),
{
"result": True
},
{
"result": True
},
{
"result": True
},
{
"result": True
},
{
"result": True
},
{
"result": True
},
{
"result": True
},
{
"result": True
},
],
'https://a.blazemeter.com/api/v4/sessions/1/stop': {},
'https://data.blazemeter.com/submit.php?session_id=1&signature=sign&test_id=1&user_id=1&pq=0&target=engine_health&update=1':
{
"result": {
'session': {}
}
}
})
obj = BlazeMeterUploader()
sent_data_points = []
obj._dpoint_serializer.get_kpi_body = get_mock(
obj._dpoint_serializer.get_kpi_body, sent_data_points)
obj.parameters['project'] = 'Proj name'
obj.settings['token'] = '123'
obj.settings['browser-open'] = 'none'
obj.engine = EngineEmul()
aggregator = ConsolidatingAggregator()
aggregator.engine = obj.engine
aggregator.settings['extend-aggregation'] = True
reader = MockReader()
watcher = MockReader()
reader.buffer_scale_idx = '100.0'
# data format: t_stamp, label, conc, r_time, con_time, latency, r_code, error, trname, byte_count
reader.data.append((1, "a", 1, 1, 1, 1, 200, None, '', 1))
reader.data.append((2, "b", 1, 2, 2, 2, 200, 'OK', '', 2))
reader.data.append((2, "b", 1, 3, 3, 3, 404, "Not Found", '', 3))
reader.data.append((2, "c", 1, 4, 4, 4, 200, None, '', 4))
reader.data.append((3, "d", 1, 5, 5, 5, 200, None, '', 5))
reader.data.append((5, "b", 1, 6, 6, 6, 200, None, '', 6))
reader.data.append((5, "c", 1, 7, 7, 7, 200, None, '', 7))
original_labels = list(d[1] for d in reader.data)
aggregator.add_underling(reader)
aggregator.add_listener(watcher)
obj.engine.aggregator = aggregator
mock.apply(obj._user)
obj._user.timeout = 0.001
obj.engine.aggregator.prepare()
obj.prepare()
obj.engine.aggregator.startup()
obj.startup()
obj.engine.aggregator.check()
obj.check()
obj.engine.aggregator.shutdown()
obj.shutdown()
obj.engine.aggregator.post_process()
obj.post_process()
sent_data_points = sent_data_points[0] + sent_data_points[1]
state_labels = [0, 1, 2]
for dp in sent_data_points:
for data in dp['cumulative'], dp['current']:
for label in data:
self.assertIn(label, original_labels + [''])
self.assertIsInstance(data[label], dict)
for key in data[label]:
self.assertIn(key, state_labels)
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 9
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertLessEqual(len(cum_dict['']['errors']), self.obj.max_error_count)
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['rtimes-len'] = 42
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(42, kpiset.rtimes_len)
for kpiset in dp['current'].values():
self.assertNotEqual(42, kpiset.rtimes_len)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
def test_set_rtimes_len(self):
obj = ConsolidatingAggregator()
obj.settings['rtimes-len'] = 42
obj.prepare()
reader = self.get_fail_reader()
obj.add_underling(reader)
listener = MockListener()
obj.add_listener(listener)
obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(42, kpiset.rtimes_len)
for kpiset in dp['current'].values():
self.assertNotEqual(42, kpiset.rtimes_len)
def test_errors_cumulative(self):
aggregator = ConsolidatingAggregator()
aggregator.track_percentiles = [50]
aggregator.prepare()
reader = self.get_fail_reader()
aggregator.add_underling(reader)
aggregator.shutdown()
aggregator.post_process()
cum_dict = aggregator.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_inf_values(self):
obj = ConsolidatingAggregator()
obj.settings['max-buffer-len'] = "inf"
obj.prepare()
self.assertEqual(obj.max_buffer_len, float("inf"))
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
self.obj.engine = EngineEmul()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
self.assertEqual(2, len(point[DataPoint.SUBRESULTS]))
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_new_aggregator(self):
# aggregator's config
self.obj.extend_aggregation = True
reader = MockReader()
watcher = MockReader()
# executor/reporter prepare level
self.obj.add_underling(reader)
self.obj.add_listener(watcher)
# send rules to underlings
self.obj.startup()
reader.buffer_scale_idx = '100.0'
# data format: t_stamp, label, conc, r_time, con_time, latency, r_code, error, trname, byte_count
reader.data.append((1, "a", 1, 1, 1, 1, 200, None, '', 0))
reader.data.append((2, "b", 1, 2, 2, 2, 200, 'OK', '', 0))
reader.data.append((2, "b", 1, 3, 3, 3, 404, "Not Found", '', 0))
reader.data.append((2, "c", 1, 4, 4, 4, 200, None, '', 0))
reader.data.append((3, "d", 1, 5, 5, 5, 200, None, '', 0))
reader.data.append((4, "b", 1, 6, 6, 6, 200, None, '', 0))
# let's collect data to seconds and send something aggregated to watcher
self.obj.shutdown()
self.obj.post_process()
data_points = watcher.results[-1][DataPoint.CUMULATIVE]
self.assertEquals(7, len(data_points))
sample_labels = {'a-0', 'b-0', 'b-1', 'b-2', 'c-0', 'd-0', ''}
self.assertEquals(sample_labels, set(data_points.keys()))
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (self.obj.generalize_labels + 1) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit + 1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels + 1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
label_count = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(label_count * 2), count=label_count)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = label_count
self.obj.add_underling(reader)
last = None
for point in self.obj.datapoints(True):
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, label_count + 1) # didn't overflow
self.assertGreaterEqual(labels_count, label_count * 0.25) # at least a quarter-filled
self.assertEqual(1, len(last[DataPoint.SUBRESULTS]))
self.assertEqual(last, last[DataPoint.SUBRESULTS][0])
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors),
self.obj.max_error_count / 2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
if point[DataPoint.SUBRESULTS] == [point]:
del point[DataPoint.SUBRESULTS]
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
def test_ramp_up_exclude(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
self.obj.engine.config['settings']['ramp-up-exclude'] = True
self.obj.engine.config['execution'] = [
{'scenario': 'first', 'ramp-up': 50},
{'scenario': 'second', 'ramp-up': '1s'},
{'scenario': 'third'}
]
self.obj.engine.config['scenarios'] = BetterDict.from_dict({
'first': {'requests': [{'url': 'first'}]},
'second': {'requests': [{'url': 'second'}]},
'third': {'requests': [{'url': 'third'}]}})
reader = get_success_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
self.assertEquals(self.obj.cumulative, {})
def test_case1(self):
mock = BZMock()
mock.mock_get.update({
'https://a.blazemeter.com/api/v4/masters/1/sessions': {
"result": {
"sessions": []
}
},
'https://a.blazemeter.com/api/v4/masters/1/full': {
"result": {
"sessions": []
}
},
'https://a.blazemeter.com/api/v4/masters/1': {
"result": {
"note": "message"
}
},
'https://a.blazemeter.com/api/v4/masters/1/status': [{
"result": {
"id": 1,
"status": "CREATE"
}
}, {
"result": {
"id": 1,
"status": "ENDED",
"progress": 101
}
}],
})
mock.mock_post = {
'https://a.blazemeter.com/api/v4/projects': {
"result": {
"id": 1
}
},
'https://a.blazemeter.com/api/v4/tests': {
"result": {
"id": 1
}
},
'https://a.blazemeter.com/api/v4/tests/1/files': {
"result": None
},
'https://a.blazemeter.com/api/v4/tests/1/start': {
"result": {
"id": 1
}
},
'https://a.blazemeter.com/api/v4/masters/1/stop': {
"result": None
},
'https://a.blazemeter.com/api/v4/masters/1/public-token': {
"result": {
"publicToken": "token"
}
},
}
mock.mock_patch = {
'https://a.blazemeter.com/api/v4/tests/1': {
"result": {}
}
}
prov = CloudProvisioning()
prov.browser_open = None
prov.public_report = True
prov.user.token = "test"
prov.engine = EngineEmul()
prov.engine.aggregator = ConsolidatingAggregator()
# prov.engine.config.merge({"modules": {"blazemeter": {"browser-open": False}}})
prov.engine.config[ScenarioExecutor.EXEC] = [{
"executor":
"mock",
"locations": {
"aws": 1
},
"files":
ModuleMock().get_resource_files()
}]
mock.apply(prov.user)
prov.prepare()
prov.startup()
prov.check()
prov._last_check_time = 0
prov.check()
prov.shutdown()
prov.post_process()
class TestConsolidatingAggregator(BZTestCase):
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
def test_two_executions(self):
self.obj.track_percentiles = [0, 50, 100]
self.obj.prepare()
underling1 = get_success_reader()
underling2 = get_success_reader()
self.obj.add_underling(underling1)
self.obj.add_underling(underling2)
cnt = 0
for _ in range(1, 10):
for point in self.obj.datapoints():
overall = point[DataPoint.CURRENT]['']
self.assertEquals(2, overall[KPISet.CONCURRENCY])
self.assertGreater(overall[KPISet.PERCENTILES]["100.0"], 0)
self.assertGreater(overall[KPISet.AVG_RESP_TIME], 0)
cnt += 1
self.assertEquals(2, cnt)
def test_errors_cumulative(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.underlings[0].cumulative
first_err_ids = [id(err) for err in cum_dict['first']['errors']]
second_err_ids = [id(err) for err in cum_dict['second']['errors']]
total_err_ids = [id(err) for err in cum_dict['']['errors']]
all_ids = first_err_ids + second_err_ids + total_err_ids
self.assertEqual(len(all_ids), len(set(all_ids)))
for label in cum_dict:
data = cum_dict[label]
total_errors_count = sum(err['cnt'] for err in data['errors'])
self.assertEqual(data['fail'], total_errors_count)
def test_labels_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_success_reader()
reader2 = get_success_reader_alot()
self.obj.log.info(len(reader1.data) + len(reader2.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
len_limit = (self.obj.generalize_labels + 1) # due to randomness, it it can go a bit higher than limit
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), len_limit + 1) # allow +1 label because '' is cumulative
def test_labels_constant_part(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_alot(prefix='http://blazedemo.com/?r=')
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertGreaterEqual(len(labels), self.obj.generalize_labels / 2) # assert that it's at least half full
self.assertLessEqual(len(labels), self.obj.generalize_labels + 1) # allow +1 label because '' is cumulative
def test_labels_aggressive_folding(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_success_reader_selected_labels()
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = 25
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
self.assertEqual(len(labels), 6)
def test_labels_aggressive_folding_2(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
LABEL_COUNT = 50
reader = get_success_reader_shrinking_labels(max_label_size=int(LABEL_COUNT * 2), count=LABEL_COUNT)
self.obj.log.info(len(reader.data))
self.obj.generalize_labels = LABEL_COUNT
self.obj.add_underling(reader)
for point in self.obj.datapoints():
last = point
cum_dict = self.obj.cumulative
labels = list(cum_dict.keys())
labels_count = len(labels)
self.assertLessEqual(labels_count, LABEL_COUNT + 1) # didn't overflow
self.assertGreaterEqual(labels_count, LABEL_COUNT * 0.25) # at least a quorter-filled
def test_errors_variety(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader1 = get_fail_reader()
reader2 = get_fail_reader_alot()
self.obj.max_error_count = 50
self.obj.add_underling(reader1)
self.obj.add_underling(reader2)
self.obj.shutdown()
self.obj.post_process()
expected = self.obj.max_error_count # due to randomness, it it can go a bit higher than limit
self.assertLessEqual(len(self.obj.known_errors), expected)
self.assertGreaterEqual(len(self.obj.known_errors), self.obj.max_error_count / 2) # assert that it's at least half full
def test_uniq_errors(self):
self.obj.track_percentiles = [50]
self.obj.prepare()
reader = get_fail_reader()
self.obj.max_error_count = 9
self.obj.add_underling(reader)
self.obj.shutdown()
self.obj.post_process()
cum_dict = self.obj.cumulative
self.assertEqual(len(cum_dict['']['errors']), 3)
def test_set_rtimes_len(self):
self.obj.settings['histogram-initial'] = 10.0
self.obj.prepare()
reader = get_fail_reader()
self.obj.add_underling(reader)
listener = MockListener()
self.obj.add_listener(listener)
self.obj.check()
for dp in listener.results:
for kpiset in dp['cumulative'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
for kpiset in dp['current'].values():
self.assertEqual(10000, kpiset[KPISet.RESP_TIMES].high)
def test_inf_values(self):
self.obj.settings['max-buffer-len'] = "inf"
self.obj.prepare()
self.assertEqual(self.obj.max_buffer_len, float("inf"))
def test_datapoint_to_json(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.obj.add_underling(get_success_reader())
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
def test_negative_response_time_scaling_crash(self):
self.obj.track_percentiles = [0.0, 50.0, 95.0, 99.0, 100.0]
self.obj.prepare()
self.sniff_log(self.obj.log)
mock = MockReader()
mock.data.append((1, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((2, "first", 1, -r(), r(), r(), 200, 'FAILx1', '', 0))
mock.data.append((5, "first", 1, -r(), r(), r(), 200, None, '', 0))
mock.data.append((7, "second", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((3, "first", 1, -r(), r(), r(), 200, 'FAILx3', '', 0))
mock.data.append((6, "second", 1, -r(), r(), r(), 200, 'unique FAIL', '', 0))
self.obj.add_underling(mock)
self.obj.check()
for point in self.obj.datapoints():
self.obj.log.info(to_json(point))
self.assertIn("Negative response time reported", self.log_recorder.warn_buff.getvalue())
def test_terminate_only(self):
"test is terminated only when it was started and didn't finished"
obj = CloudProvisioning()
obj.engine = EngineEmul()
obj.engine.config.merge({
ScenarioExecutor.EXEC: {
"executor": "mock",
"concurrency": 5500,
"locations": {
"us-east-1": 1,
"us-west": 1,
}
},
"modules": {
"mock": ModuleMock.__module__ + "." + ModuleMock.__name__
},
"provisioning": "mock"
})
obj.parameters = obj.engine.config['execution']
obj.engine.aggregator = ConsolidatingAggregator()
obj.settings["token"] = "FakeToken"
obj.settings["browser-open"] = False
obj.settings["check-interval"] = "0ms" # do not skip checks
obj.settings["use-deprecated-api"] = False
obj.client = client = BlazeMeterClientEmul(obj.log)
client.results.append({"result": []}) # find collection
client.results.append({"result": []}) # find test
client.results.append(self.__get_user_info()) # user
client.results.append({}) # upload files
client.results.append(
{"result": {
"name": "Taurus Collection",
"items": []
}}) # transform config to collection
client.results.append({"result": {"id": 42}}) # create collection
client.results.append({"result": {"id": id(obj)}}) # start
client.results.append({
"result": {
"id":
id(obj),
"sessions": [{
"id": "s1",
"status": "JMETER_CONSOLE_INIT"
}, {
"id": "s2",
"status": "JMETER_CONSOLE_INIT"
}]
}
})
client.results.append({"result": []}) # sessions
client.results.append({"result": {}}) # force start
client.results.append({"result": {
"progress": 120,
"status": "ENDED"
}}) # status should trigger shutdown
client.results.append({"result": []}) # sessions
obj.prepare()
obj.startup()
obj.check() # this one should trigger force start
self.assertTrue(obj.check())
obj.shutdown()
obj.post_process()
self.assertEqual(client.results, [])
def setUp(self):
super(TestConsolidatingAggregator, self).setUp()
self.obj = ConsolidatingAggregator()
self.obj.engine=EngineEmul()
def test_inf_values(self):
obj = ConsolidatingAggregator()
obj.settings['max-buffer-len'] = "inf"
obj.prepare()
self.assertEqual(obj.max_buffer_len, float("inf"))
