def test_locust_master(self):
self.configure({"execution": {
"concurrency": 1,
"iterations": 10,
"hold-for": 30,
"master": True,
"slaves": 1,
"scenario": {
"default-address": "http://blazedemo.com",
"script": RESOURCES_DIR + "locust/simple.py"
}
}})
self.obj.prepare()
self.obj.startup()
self.obj.get_widget()
try:
self.obj.check()
time.sleep(2)
self.obj.check()
except RuntimeError:
ROOT_LOGGER.warning("Do you use patched locust for non-GUI master?")
self.obj.shutdown()
self.obj.post_process()
self.assertFalse(self.obj.has_results())
def assertFilesEqual(expected,
actual,
replace_str="",
replace_with="",
python_files=False):
with open(expected) as exp, open(actual) as act:
act_lines = [
x.replace(replace_str, replace_with).rstrip()
for x in act.readlines()
]
exp_lines = [
x.replace(replace_str, replace_with).rstrip()
for x in exp.readlines()
]
if python_files:
act_lines = astunparse.unparse(ast.parse(
'\n'.join(act_lines))).split('\n')
exp_lines = astunparse.unparse(ast.parse(
'\n'.join(exp_lines))).split('\n')
diff = list(difflib.unified_diff(exp_lines, act_lines))
if diff:
ROOT_LOGGER.info("Replacements are: %s => %s", replace_str,
replace_with)
msg = "Failed asserting that two files are equal:\n%s\nversus\n%s\nDiff is:\n\n%s"
raise AssertionError(msg % (actual, expected, "\n".join(diff)))
def test_merge(self):
obj = Configuration()
configs = [
RESOURCES_DIR + "yaml/test.yml",
RESOURCES_DIR + "json/merge1.json",
RESOURCES_DIR + "json/merge2.json",
]
obj.load(configs)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
with open(fname) as fh:
ROOT_LOGGER.debug("JSON:\n%s", fh.read())
jmeter = obj['modules']['jmeter']
classval = jmeter['class']
self.assertEquals("bzt.modules.jmeter.JMeterExecutor", classval)
self.assertEquals("value", obj['key'])
self.assertEquals(6, len(obj["list-append"]))
self.assertEquals(2, len(obj["list-replace"]))
self.assertEquals(2, len(obj["list-replace-notexistent"]))
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertFalse("properties" in jmeter)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
checker = Configuration()
checker.load([fname])
token = checker["list-complex"][1][0]['token']
self.assertNotEquals('test', token)
token_orig = obj["list-complex"][1][0]['token']
self.assertEquals('test', token_orig)
def test_locust_master(self):
self.configure({
"execution": {
"concurrency": 1,
"iterations": 10,
"hold-for": 30,
"master": True,
"slaves": 1,
"scenario": {
"default-address": "http://blazedemo.com",
"script": RESOURCES_DIR + "locust/simple.py"
}
}
})
self.obj.prepare()
self.obj.startup()
self.obj.get_widget()
try:
self.obj.check()
time.sleep(2)
self.obj.check()
except RuntimeError:
ROOT_LOGGER.warning(
"Do you use patched locust for non-GUI master?")
self.obj.shutdown()
self.obj.post_process()
self.assertFalse(self.obj.has_results())
def test_not_junit(self):
"""
Check that JUnit runner fails if no tests were found
:return:
"""
self.configure({
EXEC: {
"executor": "selenium",
"scenario": {
"script":
RESOURCES_DIR + "selenium/invalid/NotJUnittest.java"
}
}
})
self.obj.prepare()
self.assertIsInstance(self.obj.runner, JUnitTester)
self.obj.startup()
try:
while not self.obj.check():
time.sleep(self.obj.engine.check_interval)
self.fail()
except ToolError as exc:
diagnostics = "\n".join(exc.diagnostics)
self.assertIn("Nothing to test", diagnostics)
except BaseException as exc:
ROOT_LOGGER.debug(traceback.format_exc())
self.fail("Unexpected exception %s, expected ToolError" % exc)
self.obj.shutdown()
def test_merge(self):
obj = Configuration()
configs = [
RESOURCES_DIR + "yaml/test.yml",
RESOURCES_DIR + "json/merge1.json",
RESOURCES_DIR + "json/merge2.json",
]
obj.load(configs)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
with open(fname) as fh:
ROOT_LOGGER.debug("JSON:\n%s", fh.read())
jmeter = obj['modules']['jmeter']
classval = jmeter['class']
self.assertEquals("bzt.modules.jmeter.JMeterExecutor", classval)
self.assertEquals("value", obj['key'])
self.assertEquals(6, len(obj["list-append"]))
self.assertEquals(2, len(obj["list-replace"]))
self.assertEquals(2, len(obj["list-replace-notexistent"]))
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertIsInstance(obj["list-complex"][1][0], BetterDict)
self.assertFalse("properties" in jmeter)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
checker = Configuration()
checker.load([fname])
token = checker["list-complex"][1][0]['token']
self.assertNotEquals('test', token)
token_orig = obj["list-complex"][1][0]['token']
self.assertEquals('test', token_orig)
def test_speed(self):
obj = self.obj
mock = MockReader()
mock.buffer_scale_idx = '100.0'
obj.add_listener(mock)
res = {}
# current measurements shows ~25K samples/sec
for cnt in (10, 100, 1000, 10000, 25000, 40000, 50000):
for a in range(0, cnt):
sample = (cnt, "", 1, r(1000), r(1000), r(1000), rc(), err(), '', 0)
mock.data.append(sample)
before = time.time()
for point in mock.datapoints():
pass
after = time.time()
res[cnt] = after - before
ROOT_LOGGER.info("Times: %s", res)
while mock.results:
point = mock.results.pop(0)
overall = point[DataPoint.CURRENT]['']
self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)
for point in mock.datapoints(True):
pass
def test_schedule_empty(self):
# concurrency: 1, iterations: 1
scheduler = self.get_scheduler(b("4 test\ntest\n"))
items = list(scheduler.generate())
for item in items:
ROOT_LOGGER.debug("Item: %s", item)
self.assertEqual(1, len(items))
def test_schedule_empty(self):
# concurrency: 1, iterations: 1
scheduler = self.get_scheduler(b("4 test\ntest\n"))
items = list(scheduler.generate())
for item in items:
ROOT_LOGGER.debug("Item: %s", item)
self.assertEqual(1, len(items))
def test_schedule_rps(self):
rps = 9
rampup = 12
self.configure({
"provisioning": "test",
EXEC: {
"throughput": rps,
"ramp-up": rampup,
"steps": 3,
"hold-for": 0
}
})
scheduler = self.get_scheduler(b("4 test\ntest\n"))
cnt = 0
cur = 0
currps = 0
for item in scheduler.generate():
if int(math.ceil(item[0])) != cur:
# self.assertLessEqual(currps, rps)
cur = int(math.ceil(item[0]))
ROOT_LOGGER.debug("RPS: %s", currps)
currps = 0
cnt += 1
currps += 1
ROOT_LOGGER.debug("RPS: %s", currps)
def test_speed(self):
obj = self.obj
mock = MockReader()
mock.buffer_scale_idx = '100.0'
obj.add_listener(mock)
res = {}
# current measurements shows ~25K samples/sec
for cnt in (10, 100, 1000, 10000, 25000, 40000, 50000):
for a in range(0, cnt):
sample = (cnt, "", 1, r(1000), r(1000), r(1000), rc(), err(),
'', 0)
mock.data.append(sample)
before = time.time()
for point in mock.datapoints():
pass
after = time.time()
res[cnt] = after - before
ROOT_LOGGER.info("Times: %s", res)
while mock.results:
point = mock.results.pop(0)
overall = point[DataPoint.CURRENT]['']
self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)
for point in mock.datapoints(True):
pass
def aggregated_second(self, data):
current = data[DataPoint.CURRENT]['']
ROOT_LOGGER.info("DataPoint %s: VU:%s RPS:%s/%s RT:%s",
data[DataPoint.TIMESTAMP],
current[KPISet.CONCURRENCY],
current[KPISet.SAMPLE_COUNT],
current[KPISet.FAILURES],
current[KPISet.AVG_RESP_TIME])
def assertFilesEqual(expected,
actual,
replace_str="",
replace_with="",
python_files=False):
def order(line):
line = line.replace(',', ' ,') # for imports
line = line.replace('(', '( ') # for
line = line.replace(')', ' )') # calls
line = line.split(" ")
line.sort()
return ' '.join(line)
def equal_by_content(diff):
# todo: it doesn't show diff for follow case, shouldn't we fix it?
# 01: + func1()
# 02: func2()
# 03: - func1()
# func1 moved and order has been changed
act_lines = [line[1:] for line in diff if line.startswith('-')]
exp_lines = [line[1:] for line in diff if line.startswith('+')]
for pair in zip(act_lines, exp_lines):
if order(pair[0]) != order(pair[1]):
return False
return True
if isinstance(replace_str, str):
replace_str = [replace_str]
if isinstance(replace_with, str):
replace_with = [replace_with]
with open(expected) as exp, open(actual) as act:
act_lines = act.readlines()
exp_lines = exp.readlines()
subs = dict(zip(replace_str, replace_with))
subs.update({'<': '< ', '>': ' >'}) # for xml
for key in subs:
act_lines = [x.replace(key, subs[key]).rstrip() for x in act_lines]
exp_lines = [x.replace(key, subs[key]).rstrip() for x in exp_lines]
if python_files:
act_lines = astunparse.unparse(ast.parse(
'\n'.join(act_lines))).split('\n')
exp_lines = astunparse.unparse(ast.parse(
'\n'.join(exp_lines))).split('\n')
diff = list(difflib.unified_diff(exp_lines, act_lines))
if diff and not equal_by_content(diff[2:]):
ROOT_LOGGER.info("Replacements are: %s => %s", replace_str,
replace_with)
msg = "Failed asserting that two files are equal:\n%s\nversus\n%s\nDiff is:\n\n%s"
# here we show full diff, even equal_by_content
# todo: show only really different lines
raise AssertionError(msg % (actual, expected, "\n".join(diff)))
def aggregated_second(self, data):
current = data[DataPoint.CURRENT]['']
ROOT_LOGGER.info(
"DataPoint %s: VU:%s RPS:%s/%s RT:%s",
data[DataPoint.TIMESTAMP],
current[KPISet.CONCURRENCY],
current[KPISet.SAMPLE_COUNT],
current[KPISet.FAILURES],
current[KPISet.AVG_RESP_TIME])
def test_save(self):
obj = Configuration()
obj.merge({"str": "text", "uc": six.u("ucstring")})
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.YAML)
with open(fname) as fh:
written = fh.read()
ROOT_LOGGER.debug("YAML:\n%s", written)
self.assertNotIn("unicode", written)
def test_overwrite_execution_locations(self):
obj = Configuration()
obj.merge({
"execution": [{"locations": {"us-central1-a": 1}}],
})
obj.merge({
"$execution": [{"~locations": {"harbor-1": 1}}],
})
ROOT_LOGGER.info(obj)
self.assertEqual(obj, {"execution": [{"locations": {"harbor-1": 1}}]})
def test_overwrite_execution_locations(self):
obj = Configuration()
obj.merge({
"execution": [{"locations": {"us-central1-a": 1}}],
})
obj.merge({
"$execution": [{"~locations": {"harbor-1": 1}}],
})
ROOT_LOGGER.info(obj)
self.assertEqual(obj, {"execution": [{"locations": {"harbor-1": 1}}]})
def test_server_agent(self):
obj = Monitoring()
obj.engine = EngineEmul()
obj.parameters.merge({
"server-agent": [{
"address": "127.0.0.1:4444",
"logging": "True",
"metrics": [
"cpu",
"disks"
]
}, {
"address": "10.0.0.1",
"metrics": [
"something1",
"something2"
]
}]
})
listener = LoggingMonListener()
obj.add_listener(listener)
widget = obj.get_widget()
obj.add_listener(widget)
crit_conf = BetterDict.from_dict({"condition": ">", "threshold": 5, "subject": "127.0.0.1:4444/cpu"})
criteria = MonitoringCriteria(crit_conf, obj)
obj.add_listener(criteria)
obj.client_classes = {'server-agent': ServerAgentClientEmul}
obj.prepare()
obj.startup()
for i in range(1, 10):
obj.clients[0].socket.recv_data += b("%s\t%s\t\n" % (i, i*10))
obj.check()
ROOT_LOGGER.debug("Criteria state: %s", criteria)
time.sleep(obj.engine.check_interval)
obj.shutdown()
obj.post_process()
self.assertEquals(b("test\ninterval:1\nmetrics:cpu\tdisks\nexit\n"), obj.clients[0].socket.sent_data)
if PY3:
self.assertIsNotNone(obj.clients[0].logs_file)
with open(obj.clients[0].logs_file) as serveragent_logs:
logs_reader = csv.reader(serveragent_logs)
logs_reader = list(logs_reader)
self.assertEquals(['ts', 'cpu', 'disks'], logs_reader[0])
for i in range(1, 10):
self.assertEquals([str(i), str(i * 10)], logs_reader[i][1:])
def test_save(self):
obj = Configuration()
obj.merge({
"str": "text",
"uc": six.u("ucstring")
})
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.YAML)
with open(fname) as fh:
written = fh.read()
ROOT_LOGGER.debug("YAML:\n%s", written)
self.assertNotIn("unicode", written)
def test_simple(self):
self.configure({
"provisioning": "test",
EXEC: {
"log-responses": "proto_error",
# "iterations": 5000000,
"concurrency": 10,
"throughput": 1000,
"ramp-up": "1m",
# "steps": 5,
"hold-for": "15",
"scenario": {
"timeout":
1,
"default-address":
"http://localhost:33",
"headers": {
"Connection": "close"
},
"requests": [{
"url": "/api",
"method": "POST",
"headers": {
"Content-Length": 0
},
"body": {
"param": "value"
}
}]
}
}
})
self.obj.engine.aggregator = ConsolidatingAggregator()
self.obj.engine.aggregator.engine = self.obj.engine
self.obj.engine.aggregator.add_listener(DataPointLogger())
self.obj.engine.aggregator.prepare()
self.obj.prepare()
self.obj.engine.aggregator.startup()
self.obj.startup()
while not self.obj.check():
ROOT_LOGGER.debug("Running...")
self.obj.engine.aggregator.check()
time.sleep(0.1)
self.obj.shutdown()
self.obj.engine.aggregator.shutdown()
self.obj.post_process()
self.obj.engine.aggregator.post_process()
def assertFilesEqual(expected, actual, replace_str="", replace_with="", python_files=False):
with open(expected) as exp, open(actual) as act:
act_lines = [x.replace(replace_str, replace_with).rstrip() for x in act.readlines()]
exp_lines = [x.replace(replace_str, replace_with).rstrip() for x in exp.readlines()]
if python_files:
act_lines = astunparse.unparse(ast.parse('\n'.join(act_lines))).split('\n')
exp_lines = astunparse.unparse(ast.parse('\n'.join(exp_lines))).split('\n')
diff = list(difflib.unified_diff(exp_lines, act_lines))
if diff:
ROOT_LOGGER.info("Replacements are: %s => %s", replace_str, replace_with)
msg = "Failed asserting that two files are equal:\n%s\nversus\n%s\nDiff is:\n\n%s"
raise AssertionError(msg % (actual, expected, "\n".join(diff)))
def tearDown(self):
exc, _, _ = sys.exc_info()
if exc:
try:
if hasattr(self, 'obj') and isinstance(self.obj, SelfDiagnosable):
diags = self.obj.get_error_diagnostics()
if diags:
for line in diags:
ROOT_LOGGER.info(line)
except BaseException:
pass
if self.captured_logger:
self.captured_logger.removeHandler(self.log_recorder)
self.log_recorder.close()
def tearDown(self):
exc, _, _ = sys.exc_info()
if exc:
try:
if hasattr(self, 'obj') and isinstance(self.obj, SelfDiagnosable):
diags = self.obj.get_error_diagnostics()
if diags:
for line in diags:
ROOT_LOGGER.info(line)
except BaseException:
pass
if self.captured_logger:
self.captured_logger.removeHandler(self.log_recorder)
self.log_recorder.close()
def test_simple(self):
self.configure({
"provisioning": "test",
ScenarioExecutor.EXEC: {
"log-responses": "proto_error",
# "iterations": 5000000,
"concurrency": 10,
"throughput": 1000,
"ramp-up": "1m",
# "steps": 5,
"hold-for": "15",
"scenario": {
"timeout": 1,
"default-address": "http://localhost:33",
"headers": {
"Connection": "close"
},
"requests": [
{
"url": "/api",
"method": "POST",
"headers": {
"Content-Length": 0
},
"body": {
"param": "value"}}]}}})
self.obj.engine.aggregator = ConsolidatingAggregator()
self.obj.engine.aggregator.engine = self.obj.engine
self.obj.engine.aggregator.add_listener(DataPointLogger())
self.obj.engine.aggregator.prepare()
self.obj.prepare()
self.obj.engine.aggregator.startup()
self.obj.startup()
while not self.obj.check():
ROOT_LOGGER.debug("Running...")
self.obj.engine.aggregator.check()
time.sleep(0.1)
self.obj.shutdown()
self.obj.engine.aggregator.shutdown()
self.obj.post_process()
self.obj.engine.aggregator.post_process()
def assertFilesEqual(expected,
actual,
replace_str="",
replace_with="",
python_files=False):
def equal_by_content(difference):
diff_act, diff_exp = [], []
for line in difference:
if line[0] == '-':
act_line = line[2:line.rfind('"') + 1].split(" ")[1:]
act_line.sort()
diff_act.append(act_line)
elif line[0] == '+':
diff_exp.append(line[2:line.rfind('"') + 1].split(" ")[1:])
if diff_act == diff_exp:
return True
else:
return False
if isinstance(replace_str, str):
replace_str = [replace_str]
if isinstance(replace_with, str):
replace_with = [replace_with]
with open(expected) as exp, open(actual) as act:
act_lines = act.readlines()
exp_lines = exp.readlines()
subs = dict(zip(replace_str, replace_with))
for key in subs:
act_lines = [x.replace(key, subs[key]).rstrip() for x in act_lines]
exp_lines = [x.replace(key, subs[key]).rstrip() for x in exp_lines]
if python_files:
act_lines = astunparse.unparse(ast.parse(
'\n'.join(act_lines))).split('\n')
exp_lines = astunparse.unparse(ast.parse(
'\n'.join(exp_lines))).split('\n')
diff = list(difflib.unified_diff(exp_lines, act_lines))
if diff and not equal_by_content(diff[5:]):
ROOT_LOGGER.info("Replacements are: %s => %s", replace_str,
replace_with)
msg = "Failed asserting that two files are equal:\n%s\nversus\n%s\nDiff is:\n\n%s"
raise AssertionError(msg % (actual, expected, "\n".join(diff)))
def test_binary_unicode_error(self):
fd, fname = mkstemp()
os.close(fd)
file_handler = logging.FileHandler(fname, encoding="utf-8")
file_handler.setLevel(logging.DEBUG)
ROOT_LOGGER.addHandler(file_handler)
try:
session = Session(data={'id': 1})
mock = BZMock(session)
mock.mock_post['https://data.blazemeter.com/api/v4/image/1/files?signature=None'] = {"result": 1}
with open(RESOURCES_DIR + "jmeter/jmeter-dist-2.13.zip", 'rb') as fds:
zip_content = fds.read()
session.upload_file("jtls_and_more.zip", zip_content)
finally:
ROOT_LOGGER.removeHandler(file_handler)
file_handler.close()
os.remove(fname)
def test_server_agent(self):
obj = Monitoring()
obj.engine = EngineEmul()
obj.parameters.merge({
"server-agent": [{
"address": "127.0.0.1:4444",
"metrics": ["cpu", "disks"]
}, {
"address": "10.0.0.1",
"metrics": ["something1", "something2"]
}]
})
listener = LoggingMonListener()
obj.add_listener(listener)
widget = obj.get_widget()
obj.add_listener(widget)
crit_conf = BetterDict.from_dict({
"condition": ">",
"threshold": 5,
"subject": "127.0.0.1:4444/cpu"
})
criteria = MonitoringCriteria(crit_conf, obj)
obj.add_listener(criteria)
obj.client_classes = {'server-agent': ServerAgentClientEmul}
obj.prepare()
obj.startup()
for _ in range(1, 10):
obj.clients[0].socket.recv_data += b(
"%s\t%s\n" % (random.random(), random.random()))
obj.check()
ROOT_LOGGER.debug("Criteria state: %s", criteria)
time.sleep(obj.engine.check_interval)
obj.shutdown()
obj.post_process()
self.assertEquals(b("test\ninterval:1\nmetrics:cpu\tdisks\nexit\n"),
obj.clients[0].socket.sent_data)
def test_prepare(self):
config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read())
self.configure(config['reporting'][0])
self.obj.prepare()
self.assertGreater(len(self.obj.criteria), 0)
for n in range(0, 10):
point = random_datapoint(n)
ROOT_LOGGER.info("%s: %s", n, point)
self.obj.aggregated_second(point)
try:
self.obj.check()
except AutomatedShutdown:
pass
try:
self.obj.post_process()
except AutomatedShutdown:
pass
def test_prepare(self):
config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read())
self.configure(config['reporting'][0])
self.obj.prepare()
self.assertGreater(len(self.obj.criteria), 0)
for n in range(0, 10):
point = random_datapoint(n)
ROOT_LOGGER.info("%s: %s", n, point)
self.obj.aggregated_second(point)
try:
self.obj.check()
except AutomatedShutdown:
pass
try:
self.obj.post_process()
except AutomatedShutdown:
pass
def test_schedule_rps(self):
self.obj.engine.config.merge({"provisioning": "test"})
rps = 9
rampup = 12
self.obj.execution.merge({"throughput": rps, "ramp-up": rampup, "steps": 3, "hold-for": 0})
scheduler = self.get_scheduler(b("4 test\ntest\n"))
cnt = 0
cur = 0
currps = 0
for item in scheduler.generate():
if int(math.ceil(item[0])) != cur:
# self.assertLessEqual(currps, rps)
cur = int(math.ceil(item[0]))
ROOT_LOGGER.debug("RPS: %s", currps)
currps = 0
cnt += 1
currps += 1
ROOT_LOGGER.debug("RPS: %s", currps)
def test_stdev_performance(self):
start = time.time()
self.configure(RESOURCES_DIR + "jmeter/jtl/slow-stdev.jtl")
res = list(self.obj.datapoints(final_pass=True))
lst_json = to_json(res)
self.assertNotIn('"perc": {},', lst_json)
elapsed = time.time() - start
ROOT_LOGGER.debug("Elapsed/per datapoint: %s / %s", elapsed,
elapsed / len(res))
# self.assertLess(elapsed, len(res)) # less than 1 datapoint per sec is a no-go
exp = [
0.53060066889723, 0.39251356581014, 0.388405157629,
0.38927586980868, 0.30511697736531, 0.21160424043633,
0.07339064994943
]
self.assertEqual(exp, [
round(x[DataPoint.CURRENT][''][KPISet.STDEV_RESP_TIME], 14)
for x in res
])
def test_stdev_performance(self):
start = time.time()
self.configure(RESOURCES_DIR + "/jmeter/jtl/slow-stdev.jtl")
res = list(self.obj.datapoints(final_pass=True))
lst_json = to_json(res)
self.assertNotIn('"perc": {},', lst_json)
elapsed = time.time() - start
ROOT_LOGGER.debug("Elapsed/per datapoint: %s / %s", elapsed,
elapsed / len(res))
# self.assertLess(elapsed, len(res)) # less than 1 datapoint per sec is a no-go
exp = [
2.2144798867972773, 0.7207704268609725, 0.606834452578833,
0.8284089170237546, 0.5858142211763572, 0.622922628329711,
0.5529488620851849, 0.6933748292117727, 0.4876162181858197,
0.42471180222446503, 0.2512251128133865
]
self.assertEqual(
exp,
[x[DataPoint.CURRENT][''][KPISet.STDEV_RESP_TIME] for x in res])
def test_public_report(self):
mock = BZMock()
mock.mock_get.update({
'https://a.blazemeter.com/api/v4/tests?workspaceId=1&name=Taurus+Test': {"result": []}
})
mock.mock_post.update({
'https://a.blazemeter.com/api/v4/projects': {"result": {'id': 1}},
'https://a.blazemeter.com/api/v4/tests': {'result': {'id': 'unittest1'}},
'https://a.blazemeter.com/api/v4/tests/unittest1/start-external': {"result": {
'session': {'id': 'sess1', 'userId': 1, 'testId': 1},
'master': {'id': 'master1', 'userId': 1},
'signature': ''
}},
'https://a.blazemeter.com/api/v4/masters/master1/public-token': {'result': {'publicToken': 'publicToken'}},
'https://data.blazemeter.com/submit.php?session_id=sess1&signature=&test_id=1&user_id=1&pq=0&target=labels_bulk&update=1': {
"result": {'session': {}}},
'https://data.blazemeter.com/api/v4/image/sess1/files?signature=': {'result': True},
})
obj = BlazeMeterUploader()
obj.settings['token'] = '123'
obj.settings['browser-open'] = 'none'
obj.settings['public-report'] = True
obj.settings['send-monitoring'] = False
obj.engine = EngineEmul()
mock.apply(obj._user)
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.aggregated_second(random_datapoint(10))
obj.check()
obj.shutdown()
obj.post_process()
log_buff = self.log_recorder.info_buff.getvalue()
log_line = "Public report link: https://a.blazemeter.com/app/?public-token=publicToken#/masters/master1/summary"
self.assertIn(log_line, log_buff)
ROOT_LOGGER.warning("\n".join([x['url'] for x in mock.requests]))
self.assertEqual(14, len(mock.requests))
def test_stdev_performance(self):
start = time.time()
self.configure(RESOURCES_DIR + "jmeter/jtl/slow-stdev.jtl")
res = list(self.obj.datapoints(final_pass=True))
lst_json = to_json(res)
self.assertNotIn('"perc": {},', lst_json)
elapsed = time.time() - start
ROOT_LOGGER.debug("Elapsed/per datapoint: %s / %s", elapsed,
elapsed / len(res))
# self.assertLess(elapsed, len(res)) # less than 1 datapoint per sec is a no-go
exp = [
0.53060066889723, 0.39251356581014, 0.388405157629,
0.52855748890714, 0.39107758224016, 0.38999119030886,
0.32537625773864, 0.47057465198195, 0.2746790136753,
0.23251104555698, 0.08369447671202
]
self.assertEqual(exp, [
round(x[DataPoint.CURRENT][''][KPISet.STDEV_RESP_TIME], 14)
for x in res
])
def assertFilesEqual(self,
expected,
actual,
replace_str="",
replace_with=""):
# import shutil; shutil.copy(actual, expected)
with open(expected) as exp, open(actual) as act:
act_lines = [
x.replace(replace_str, replace_with).rstrip()
for x in act.readlines()
]
exp_lines = [
x.replace(replace_str, replace_with).rstrip()
for x in exp.readlines()
]
diff = list(difflib.unified_diff(exp_lines, act_lines))
if diff:
ROOT_LOGGER.info("Replacements are: %s => %s", replace_str,
replace_with)
msg = "Failed asserting that two files are equal:\n" + actual + "\nversus\n" + expected + "\nDiff is:\n"
raise AssertionError(msg + "\n".join(diff))
def test_stdev_performance(self):
start = time.time()
self.configure(RESOURCES_DIR + "/jmeter/jtl/slow-stdev.jtl")
res = list(self.obj.datapoints(final_pass=True))
lst_json = to_json(res)
self.assertNotIn('"perc": {},', lst_json)
elapsed = time.time() - start
ROOT_LOGGER.debug("Elapsed/per datapoint: %s / %s", elapsed, elapsed / len(res))
# self.assertLess(elapsed, len(res)) # less than 1 datapoint per sec is a no-go
exp = [2.2144798867972773,
0.7207704268609725,
0.606834452578833,
0.8284089170237546,
0.5858142211763572,
0.622922628329711,
0.5529488620851849,
0.6933748292117727,
0.4876162181858197,
0.42471180222446503,
0.2512251128133865]
self.assertEqual(exp, [x[DataPoint.CURRENT][''][KPISet.STDEV_RESP_TIME] for x in res])
def test_not_junit(self):
"""
Check that JUnit runner fails if no tests were found
:return:
"""
self.configure({
ScenarioExecutor.EXEC: {
"executor": "selenium",
"scenario": {"script": RESOURCES_DIR + "selenium/invalid/NotJUnittest.java"}}})
self.obj.prepare()
self.assertIsInstance(self.obj.runner, JUnitTester)
self.obj.startup()
try:
while not self.obj.check():
time.sleep(self.obj.engine.check_interval)
self.fail()
except ToolError as exc:
diagnostics = "\n".join(exc.diagnostics)
self.assertIn("Nothing to test", diagnostics)
except BaseException as exc:
ROOT_LOGGER.debug(traceback.format_exc())
self.fail("Unexpected exception %s, expected ToolError" % exc)
self.obj.shutdown()
def test_load(self):
obj = Configuration()
configs = [
BASE_CONFIG, RESOURCES_DIR + "json/jmx.json",
RESOURCES_DIR + "json/concurrency.json"
]
obj.load(configs)
ROOT_LOGGER.debug("config:\n%s", obj)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
with open(fname) as fh:
ROOT_LOGGER.debug("JSON:\n%s", fh.read())
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.YAML)
with open(fname) as fh:
ROOT_LOGGER.debug("YAML:\n%s", fh.read())
def test_load(self):
obj = Configuration()
configs = [
BASE_CONFIG,
RESOURCES_DIR + "json/jmx.json",
RESOURCES_DIR + "json/concurrency.json"
]
obj.load(configs)
ROOT_LOGGER.debug("config:\n%s", obj)
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.JSON)
with open(fname) as fh:
ROOT_LOGGER.debug("JSON:\n%s", fh.read())
fname = tempfile.mkstemp()[1]
obj.dump(fname, Configuration.YAML)
with open(fname) as fh:
ROOT_LOGGER.debug("YAML:\n%s", fh.read())
def check_schedule_size_estimate(self, execution):
self.configure({
ScenarioExecutor.EXEC: execution,
"provisioning": "local",
})
load = self.obj.get_load()
self.obj.generator = TaurusPBenchGenerator(self.obj, ROOT_LOGGER)
self.obj.generator.generate_payload(self.obj.get_scenario())
payload_count = len(self.obj.get_scenario().get('requests', []))
sch = Scheduler(load, self.obj.generator.payload_file, ROOT_LOGGER)
estimated_schedule_size = self.obj.generator._estimate_schedule_size(load, payload_count)
ROOT_LOGGER.debug("Estimated schedule size: %s", estimated_schedule_size)
items = list(sch.generate())
actual_schedule_size = len(items)
ROOT_LOGGER.debug("Actual schedule size: %s", actual_schedule_size)
if actual_schedule_size != 0:
error = abs(estimated_schedule_size - actual_schedule_size)
error_rel = error / float(actual_schedule_size)
ROOT_LOGGER.debug("Estimation error: %s", error)
if error_rel >= 0.1:
self.fail("Estimation failed (error=%s) on config %s" % (error_rel, pprint.pformat(execution)))
def check_schedule_size_estimate(self, execution):
self.configure({
ScenarioExecutor.EXEC: execution,
"provisioning": "local",
})
load = self.obj.get_load()
self.obj.generator = TaurusPBenchGenerator(self.obj, ROOT_LOGGER)
self.obj.generator.generate_payload(self.obj.get_scenario())
payload_count = len(self.obj.get_scenario().get('requests', []))
sch = Scheduler(load, self.obj.generator.payload_file, ROOT_LOGGER)
estimated_schedule_size = self.obj.generator._estimate_schedule_size(load, payload_count)
ROOT_LOGGER.debug("Estimated schedule size: %s", estimated_schedule_size)
items = list(sch.generate())
actual_schedule_size = len(items)
ROOT_LOGGER.debug("Actual schedule size: %s", actual_schedule_size)
if actual_schedule_size != 0:
error = abs(estimated_schedule_size - actual_schedule_size)
error_rel = error / float(actual_schedule_size)
ROOT_LOGGER.debug("Estimation error: %s", error)
if error_rel >= 0.1:
self.fail("Estimation failed (error=%s) on config %s" % (error_rel, pprint.pformat(execution)))
def test_xml_format_passfail(self):
obj = JUnitXMLReporter()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
obj.engine.provisioning = CloudProvisioning()
obj.engine.provisioning.results_url = "http://test/report/123"
pass_fail1 = CriteriaProcessor([], None)
crit_cfg1 = BetterDict()
crit_cfg2 = BetterDict()
crit_cfg3 = BetterDict()
crit_cfg4 = BetterDict()
crit_cfg1.merge({
'stop': True, 'label': 'Sample 1 Triggered', 'fail': True,
'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt'})
crit_cfg2.merge({
'stop': True, 'label': 'Sample 1 Not Triggered', 'fail': True,
'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt'})
crit_cfg3.merge({
'stop': True, 'label': 'Sample 2 Triggered', 'fail': True, 'timeframe': -1,
'threshold': '150ms', 'condition': '<=', 'subject': 'avg-rt'})
crit_cfg4.merge({
'stop': True, 'label': 'Sample 2 Not Triggered', 'fail': True,
'timeframe': -1, 'threshold': '300ms', 'condition': '=', 'subject': 'avg-rt'})
fc1_triggered = DataCriterion(crit_cfg1, pass_fail1)
fc1_not_triggered = DataCriterion(crit_cfg2, pass_fail1)
pass_fail2 = CriteriaProcessor([], None)
fc2_triggered = DataCriterion(crit_cfg3, pass_fail1)
fc2_not_triggered = DataCriterion(crit_cfg4, pass_fail1)
pass_fail1.criteria.append(fc1_triggered)
pass_fail1.criteria.append(fc1_not_triggered)
pass_fail2.criteria.append(fc2_triggered)
pass_fail2.criteria.append(fc2_not_triggered)
fc1_triggered.is_triggered = True
fc2_triggered.is_triggered = True
pass_fail = PassFailStatus()
pass_fail.processors.append(pass_fail1)
pass_fail.processors.append(pass_fail2)
obj.engine.reporters.append(pass_fail)
obj.engine.reporters.append(BlazeMeterUploader())
path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml_passfail', dir=obj.engine.artifacts_dir)
obj.parameters.merge({"filename": path_from_config, "data-source": "pass-fail"})
obj.prepare()
obj.last_second = DataPoint(0)
obj.post_process()
with open(obj.report_file_path, 'rb') as fds:
f_contents = fds.read()
ROOT_LOGGER.info("File: %s", f_contents)
xml_tree = etree.fromstring(f_contents)
self.assertEqual('testsuites', xml_tree.tag)
suite = xml_tree.getchildren()[0]
self.assertEqual('testsuite', suite.tag)
test_cases = suite.getchildren()
self.assertEqual(4, len(test_cases))
self.assertEqual('testcase', test_cases[0].tag)
self.assertEqual('error', test_cases[0].getchildren()[1].tag)
self.assertEqual('error', test_cases[2].getchildren()[1].tag)
sys_out = test_cases[0].getchildren()[0]
self.assertEqual('system-out', sys_out.tag)
self.assertIn('BlazeMeter report link: http://test/report/123', sys_out.text)
def test_xml_format_sample_labels(self):
# generate xml, compare hash
obj = JUnitXMLReporter()
obj.engine = EngineEmul()
rep = BlazeMeterUploader()
rep.results_url = "http://report/123"
obj.engine.reporters.append(rep)
path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml-sample-labels',
dir=obj.engine.artifacts_dir)
# data-source: finalstats by default
obj.parameters = BetterDict.from_dict({"filename": path_from_config})
obj.prepare()
datapoint = DataPoint(0, [])
cumul_data = datapoint[DataPoint.CUMULATIVE]
cumul_data[""] = KPISet.from_dict({
KPISet.AVG_CONN_TIME: 7.890211417203362e-06,
KPISet.RESP_TIMES: Counter({
0.0: 32160, 0.001: 24919, 0.002: 1049, 0.003: 630, 0.004: 224, 0.005: 125,
0.006: 73, 0.007: 46, 0.008: 32, 0.009: 20, 0.011: 8, 0.01: 8, 0.017: 3,
0.016: 3, 0.014: 3, 0.013: 3, 0.04: 2, 0.012: 2, 0.079: 1, 0.081: 1,
0.019: 1, 0.015: 1}),
KPISet.ERRORS: [{'msg': 'Forbidden', 'cnt': 7300, 'type': 0,
'urls': Counter({'http://192.168.1.1/anotherquery': 7300}),
KPISet.RESP_CODES: '403'},
{'msg': 'Assertion failed: text /smth/ not found', 'cnt': 73, 'type': 1,
'urls': Counter({'http://192.168.1.1/anotherquery': 73}),
KPISet.RESP_CODES: '200'},
],
KPISet.STDEV_RESP_TIME: 0.04947974228872108,
KPISet.AVG_LATENCY: 0.0002825639815220692,
KPISet.RESP_CODES: Counter({'304': 29656, '403': 29656, '200': 2}),
KPISet.PERCENTILES: {'95.0': 0.001, '0.0': 0.0, '99.9': 0.008, '90.0': 0.001,
'100.0': 0.081, '99.0': 0.003, '50.0': 0.0},
KPISet.SUCCESSES: 29658,
KPISet.SAMPLE_COUNT: 59314,
KPISet.CONCURRENCY: 0,
KPISet.AVG_RESP_TIME: 0.0005440536804127192,
KPISet.FAILURES: 29656})
cumul_data["http://192.168.1.1/somequery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME: 9.609548856969457e-06,
KPISet.RESP_TIMES: Counter({
0.0: 17219, 0.001: 11246, 0.002: 543, 0.003: 341,
0.004: 121,
0.005: 66, 0.006: 36, 0.007: 33, 0.008: 18,
0.009: 12, 0.011: 6,
0.01: 5, 0.013: 2, 0.017: 2, 0.012: 2, 0.079: 1,
0.016: 1,
0.014: 1, 0.019: 1, 0.04: 1, 0.081: 1}),
KPISet.ERRORS: [],
KPISet.STDEV_RESP_TIME: 0.04073402130687656,
KPISet.AVG_LATENCY: 1.7196034796682178e-06,
KPISet.RESP_CODES: Counter({'304': 29656, '200': 2}),
KPISet.PERCENTILES: {'95.0': 0.001, '0.0': 0.0,
'99.9': 0.009,
'90.0': 0.001,
'100.0': 0.081,
'99.0': 0.004,
'50.0': 0.0},
KPISet.SUCCESSES: 29658,
KPISet.SAMPLE_COUNT: 29658,
KPISet.CONCURRENCY: 0,
KPISet.AVG_RESP_TIME: 0.0005164542450603551, KPISet.FAILURES: 0})
cumul_data["http://192.168.1.1/anotherquery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME: 6.1707580253574335e-06,
KPISet.RESP_TIMES: Counter({0.0: 14941, 0.001: 13673, 0.002: 506,
0.003: 289, 0.004: 103,
0.005: 59, 0.006: 37, 0.008: 14,
0.007: 13, 0.009: 8, 0.01: 3,
0.011: 2, 0.016: 2, 0.014: 2,
0.017: 1, 0.013: 1, 0.015: 1,
0.04: 1}),
KPISet.ERRORS: [
{'msg': 'Forbidden', 'cnt': 7300, 'type': 0,
'urls': Counter({'http://192.168.1.1/anotherquery': 7300}),
KPISet.RESP_CODES: '403'},
{'msg': 'Assertion failed: text /smth/ not found', 'cnt': 73, 'type': 1,
'urls': Counter({'http://192.168.1.1/anotherquery': 73}),
KPISet.RESP_CODES: '200'},
],
KPISet.STDEV_RESP_TIME: 0.032465137860758844,
KPISet.AVG_LATENCY: 0.0005634272997032645,
KPISet.RESP_CODES: Counter({'403': 29656}),
KPISet.PERCENTILES: {'95.0': 0.001, '0.0': 0.0,
'99.9': 0.008, '90.0': 0.001,
'100.0': 0.04, '99.0': 0.003,
'50.0': 0.0},
KPISet.SUCCESSES: 0,
KPISet.SAMPLE_COUNT: 29656,
KPISet.CONCURRENCY: 0,
KPISet.AVG_RESP_TIME: 0.0005716549770704078,
KPISet.FAILURES: 29656})
cumul_data["http://192.168.100.100/somequery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME: 9.609548856969457e-06,
KPISet.RESP_TIMES: Counter({
0.0: 17219, 0.001: 11246, 0.002: 543,
0.003: 341, 0.004: 121,
0.005: 66, 0.006: 36, 0.007: 33, 0.008: 18,
0.009: 12, 0.011: 6,
0.01: 5, 0.013: 2, 0.017: 2, 0.012: 2,
0.079: 1, 0.016: 1,
0.014: 1, 0.019: 1, 0.04: 1, 0.081: 1}),
KPISet.ERRORS: [],
KPISet.STDEV_RESP_TIME: 0.04073402130687656,
KPISet.AVG_LATENCY: 1.7196034796682178e-06,
KPISet.RESP_CODES: Counter({'304': 29656, '200': 2}),
KPISet.PERCENTILES: {'95.0': 0.001, '0.0': 0.0,
'99.9': 0.009, '90.0': 0.001,
'100.0': 0.081, '99.0': 0.004,
'50.0': 0.0},
KPISet.SUCCESSES: 29658,
KPISet.SAMPLE_COUNT: 29658,
KPISet.CONCURRENCY: 0,
KPISet.AVG_RESP_TIME: 0.0005164542450603551,
KPISet.FAILURES: 0})
obj.aggregated_second(datapoint)
obj.post_process()
with open(obj.report_file_path, 'rb') as fds:
f_contents = fds.read()
ROOT_LOGGER.info("File: %s", f_contents)
xml_tree = etree.fromstring(f_contents)
self.assertEqual('testsuites', xml_tree.tag)
suite = xml_tree.getchildren()[0]
self.assertEqual('testsuite', suite.tag)
self.assertListEqual(['sample_labels', "bzt"], suite.values())
test_cases = suite.getchildren()
self.assertEqual(3, len(test_cases))
self.assertEqual('testcase', test_cases[0].tag)
self.assertEqual('error', test_cases[0].getchildren()[1].tag)
self.assertEqual('failure', test_cases[0].getchildren()[2].tag)
self.assertEqual('system-out', test_cases[0].getchildren()[0].tag)
self.assertIn('BlazeMeter report link: http://report/123', test_cases[0].getchildren()[0].text)
def monitoring_data(self, data):
ROOT_LOGGER.debug("Data: %s", data)
def test_xml_format_sample_labels(self):
# generate xml, compare hash
obj = JUnitXMLReporter()
obj.engine = EngineEmul()
rep = BlazeMeterUploader()
rep.results_url = "http://report/123"
obj.engine.reporters.append(rep)
path_from_config = tempfile.mktemp(suffix='.xml',
prefix='junit-xml-sample-labels',
dir=obj.engine.artifacts_dir)
# data-source: finalstats by default
obj.parameters = BetterDict.from_dict({"filename": path_from_config})
obj.prepare()
datapoint = DataPoint(0, [])
cumul_data = datapoint[DataPoint.CUMULATIVE]
cumul_data[""] = KPISet.from_dict({
KPISet.AVG_CONN_TIME:
7.890211417203362e-06,
KPISet.RESP_TIMES:
Counter({
0.0: 32160,
0.001: 24919,
0.002: 1049,
0.003: 630,
0.004: 224,
0.005: 125,
0.006: 73,
0.007: 46,
0.008: 32,
0.009: 20,
0.011: 8,
0.01: 8,
0.017: 3,
0.016: 3,
0.014: 3,
0.013: 3,
0.04: 2,
0.012: 2,
0.079: 1,
0.081: 1,
0.019: 1,
0.015: 1
}),
KPISet.ERRORS: [
{
'msg': 'Forbidden',
'cnt': 7300,
'type': 0,
'urls': Counter({'http://192.168.1.1/anotherquery': 7300}),
KPISet.RESP_CODES: '403'
},
{
'msg': 'Assertion failed: text /smth/ not found',
'cnt': 73,
'type': 1,
'urls': Counter({'http://192.168.1.1/anotherquery': 73}),
KPISet.RESP_CODES: '200'
},
],
KPISet.STDEV_RESP_TIME:
0.04947974228872108,
KPISet.AVG_LATENCY:
0.0002825639815220692,
KPISet.RESP_CODES:
Counter({
'304': 29656,
'403': 29656,
'200': 2
}),
KPISet.PERCENTILES: {
'95.0': 0.001,
'0.0': 0.0,
'99.9': 0.008,
'90.0': 0.001,
'100.0': 0.081,
'99.0': 0.003,
'50.0': 0.0
},
KPISet.SUCCESSES:
29658,
KPISet.SAMPLE_COUNT:
59314,
KPISet.CONCURRENCY:
0,
KPISet.AVG_RESP_TIME:
0.0005440536804127192,
KPISet.FAILURES:
29656
})
cumul_data["http://192.168.1.1/somequery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME:
9.609548856969457e-06,
KPISet.RESP_TIMES:
Counter({
0.0: 17219,
0.001: 11246,
0.002: 543,
0.003: 341,
0.004: 121,
0.005: 66,
0.006: 36,
0.007: 33,
0.008: 18,
0.009: 12,
0.011: 6,
0.01: 5,
0.013: 2,
0.017: 2,
0.012: 2,
0.079: 1,
0.016: 1,
0.014: 1,
0.019: 1,
0.04: 1,
0.081: 1
}),
KPISet.ERRORS: [],
KPISet.STDEV_RESP_TIME:
0.04073402130687656,
KPISet.AVG_LATENCY:
1.7196034796682178e-06,
KPISet.RESP_CODES:
Counter({
'304': 29656,
'200': 2
}),
KPISet.PERCENTILES: {
'95.0': 0.001,
'0.0': 0.0,
'99.9': 0.009,
'90.0': 0.001,
'100.0': 0.081,
'99.0': 0.004,
'50.0': 0.0
},
KPISet.SUCCESSES:
29658,
KPISet.SAMPLE_COUNT:
29658,
KPISet.CONCURRENCY:
0,
KPISet.AVG_RESP_TIME:
0.0005164542450603551,
KPISet.FAILURES:
0
})
cumul_data["http://192.168.1.1/anotherquery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME:
6.1707580253574335e-06,
KPISet.RESP_TIMES:
Counter({
0.0: 14941,
0.001: 13673,
0.002: 506,
0.003: 289,
0.004: 103,
0.005: 59,
0.006: 37,
0.008: 14,
0.007: 13,
0.009: 8,
0.01: 3,
0.011: 2,
0.016: 2,
0.014: 2,
0.017: 1,
0.013: 1,
0.015: 1,
0.04: 1
}),
KPISet.ERRORS: [
{
'msg': 'Forbidden',
'cnt': 7300,
'type': 0,
'urls': Counter({'http://192.168.1.1/anotherquery': 7300}),
KPISet.RESP_CODES: '403'
},
{
'msg': 'Assertion failed: text /smth/ not found',
'cnt': 73,
'type': 1,
'urls': Counter({'http://192.168.1.1/anotherquery': 73}),
KPISet.RESP_CODES: '200'
},
],
KPISet.STDEV_RESP_TIME:
0.032465137860758844,
KPISet.AVG_LATENCY:
0.0005634272997032645,
KPISet.RESP_CODES:
Counter({'403': 29656}),
KPISet.PERCENTILES: {
'95.0': 0.001,
'0.0': 0.0,
'99.9': 0.008,
'90.0': 0.001,
'100.0': 0.04,
'99.0': 0.003,
'50.0': 0.0
},
KPISet.SUCCESSES:
0,
KPISet.SAMPLE_COUNT:
29656,
KPISet.CONCURRENCY:
0,
KPISet.AVG_RESP_TIME:
0.0005716549770704078,
KPISet.FAILURES:
29656
})
cumul_data["http://192.168.100.100/somequery"] = KPISet.from_dict({
KPISet.AVG_CONN_TIME:
9.609548856969457e-06,
KPISet.RESP_TIMES:
Counter({
0.0: 17219,
0.001: 11246,
0.002: 543,
0.003: 341,
0.004: 121,
0.005: 66,
0.006: 36,
0.007: 33,
0.008: 18,
0.009: 12,
0.011: 6,
0.01: 5,
0.013: 2,
0.017: 2,
0.012: 2,
0.079: 1,
0.016: 1,
0.014: 1,
0.019: 1,
0.04: 1,
0.081: 1
}),
KPISet.ERRORS: [],
KPISet.STDEV_RESP_TIME:
0.04073402130687656,
KPISet.AVG_LATENCY:
1.7196034796682178e-06,
KPISet.RESP_CODES:
Counter({
'304': 29656,
'200': 2
}),
KPISet.PERCENTILES: {
'95.0': 0.001,
'0.0': 0.0,
'99.9': 0.009,
'90.0': 0.001,
'100.0': 0.081,
'99.0': 0.004,
'50.0': 0.0
},
KPISet.SUCCESSES:
29658,
KPISet.SAMPLE_COUNT:
29658,
KPISet.CONCURRENCY:
0,
KPISet.AVG_RESP_TIME:
0.0005164542450603551,
KPISet.FAILURES:
0
})
obj.aggregated_second(datapoint)
obj.post_process()
with open(obj.report_file_path, 'rb') as fds:
f_contents = fds.read()
ROOT_LOGGER.info("File: %s", f_contents)
xml_tree = etree.fromstring(f_contents)
self.assertEqual('testsuites', xml_tree.tag)
suite = xml_tree.getchildren()[0]
self.assertEqual('testsuite', suite.tag)
self.assertListEqual(['sample_labels', "bzt"], suite.values())
test_cases = suite.getchildren()
self.assertEqual(3, len(test_cases))
self.assertEqual('testcase', test_cases[0].tag)
self.assertEqual('error', test_cases[0].getchildren()[1].tag)
self.assertEqual('failure', test_cases[0].getchildren()[2].tag)
self.assertEqual('system-out', test_cases[0].getchildren()[0].tag)
self.assertIn('BlazeMeter report link: http://report/123',
test_cases[0].getchildren()[0].text)
def test_xml_format_passfail(self):
obj = JUnitXMLReporter()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
obj.engine.provisioning = CloudProvisioning()
obj.engine.provisioning.results_url = "http://test/report/123"
pass_fail1 = CriteriaProcessor([], None)
crit_cfg1 = BetterDict()
crit_cfg2 = BetterDict()
crit_cfg3 = BetterDict()
crit_cfg4 = BetterDict()
crit_cfg1.merge({
'stop': True,
'label': 'Sample 1 Triggered',
'fail': True,
'timeframe': -1,
'threshold': '150ms',
'condition': '<',
'subject': 'avg-rt'
})
crit_cfg2.merge({
'stop': True,
'label': 'Sample 1 Not Triggered',
'fail': True,
'timeframe': -1,
'threshold': '300ms',
'condition': '>',
'subject': 'avg-rt'
})
crit_cfg3.merge({
'stop': True,
'label': 'Sample 2 Triggered',
'fail': True,
'timeframe': -1,
'threshold': '150ms',
'condition': '<=',
'subject': 'avg-rt'
})
crit_cfg4.merge({
'stop': True,
'label': 'Sample 2 Not Triggered',
'fail': True,
'timeframe': -1,
'threshold': '300ms',
'condition': '=',
'subject': 'avg-rt'
})
fc1_triggered = DataCriterion(crit_cfg1, pass_fail1)
fc1_not_triggered = DataCriterion(crit_cfg2, pass_fail1)
pass_fail2 = CriteriaProcessor([], None)
fc2_triggered = DataCriterion(crit_cfg3, pass_fail1)
fc2_not_triggered = DataCriterion(crit_cfg4, pass_fail1)
pass_fail1.criteria.append(fc1_triggered)
pass_fail1.criteria.append(fc1_not_triggered)
pass_fail2.criteria.append(fc2_triggered)
pass_fail2.criteria.append(fc2_not_triggered)
fc1_triggered.is_triggered = True
fc2_triggered.is_triggered = True
pass_fail = PassFailStatus()
pass_fail.processors.append(pass_fail1)
pass_fail.processors.append(pass_fail2)
obj.engine.reporters.append(pass_fail)
obj.engine.reporters.append(BlazeMeterUploader())
path_from_config = tempfile.mktemp(suffix='.xml',
prefix='junit-xml_passfail',
dir=obj.engine.artifacts_dir)
obj.parameters.merge({
"filename": path_from_config,
"data-source": "pass-fail"
})
obj.prepare()
obj.last_second = DataPoint(0)
obj.post_process()
with open(obj.report_file_path, 'rb') as fds:
f_contents = fds.read()
ROOT_LOGGER.info("File: %s", f_contents)
xml_tree = etree.fromstring(f_contents)
self.assertEqual('testsuites', xml_tree.tag)
suite = xml_tree.getchildren()[0]
self.assertEqual('testsuite', suite.tag)
test_cases = suite.getchildren()
self.assertEqual(4, len(test_cases))
self.assertEqual('testcase', test_cases[0].tag)
self.assertEqual('error', test_cases[0].getchildren()[1].tag)
self.assertEqual('error', test_cases[2].getchildren()[1].tag)
sys_out = test_cases[0].getchildren()[0]
self.assertEqual('system-out', sys_out.tag)
self.assertIn('BlazeMeter report link: http://test/report/123',
sys_out.text)
