def test_log_messages_percentiles(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.parameters.merge({
"failed-labels": False,
"percentiles": True,
"summary": False,
"test-duration": False
})
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
target_output = (
"Average times: total 0.001, latency 0.000, connect 0.000\n"
"Percentile 0.0%: 0.000\n"
"Percentile 50.0%: 0.000\n"
"Percentile 90.0%: 0.001\n"
"Percentile 95.0%: 0.001\n"
"Percentile 99.0%: 0.003\n"
"Percentile 99.9%: 0.008\n"
"Percentile 100.0%: 0.081\n")
self.assertEqual(target_output, self.log_recorder.info_buff.getvalue())
def test_log_messages_percentiles(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": False, "percentiles": True, "summary": False,
"test-duration": False, "summary-labels": False})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
target_output = ("Average times: total 0.001, latency 0.000, connect 0.000\n"
"Percentiles:\n"
"+---------------+---------------+\n"
"| Percentile, % | Resp. Time, s |\n"
"+---------------+---------------+\n"
"| 0.0 | 0.0 |\n"
"| 50.0 | 0.0 |\n"
"| 90.0 | 0.001 |\n"
"| 95.0 | 0.001 |\n"
"| 99.0 | 0.003 |\n"
"| 99.9 | 0.008 |\n"
"| 100.0 | 0.081 |\n"
"+---------------+---------------+\n"
)
self.assertEqual(target_output, self.log_recorder.info_buff.getvalue())
def test_log_messages_percentiles(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": False, "percentiles": True, "summary": False, "test-duration": False,
"summary-labels": False})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
target_output = ("Average times: total 0.001, latency 0.000, connect 0.000\n"
"Percentiles:\n"
"+---------------+---------------+\n"
"| Percentile, % | Resp. Time, s |\n"
"+---------------+---------------+\n"
"| 0.0 | 0.0 |\n"
"| 50.0 | 0.0 |\n"
"| 90.0 | 0.001 |\n"
"| 95.0 | 0.001 |\n"
"| 99.0 | 0.003 |\n"
"| 99.9 | 0.008 |\n"
"| 100.0 | 0.081 |\n"
"+---------------+---------------+\n"
)
self.assertEqual(target_output, self.log_recorder.info_buff.getvalue())
def test_csv_report_fieldname_order(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
csv_report = obj.engine.create_artifact("report", ".csv")
obj.parameters.merge({
"dump-csv": csv_report,
})
obj.startup()
obj.aggregated_second(self.__get_datapoint(ts=90))
obj.aggregated_second(self.__get_datapoint(ts=100))
obj.shutdown()
obj.post_process()
self.assertTrue(os.path.exists(csv_report))
with open(csv_report) as fds:
fieldnames = fds.readline().strip().split(",")
perc_fields = [
float(name[5:]) for name in fieldnames if name.startswith('perc_')
]
self.assertTrue(sorted(perc_fields) == perc_fields)
rc_fields = [
float(name[3:]) for name in fieldnames if name.startswith('rc_')
]
self.assertTrue(sorted(rc_fields) == rc_fields)
def test_log_messages_summary_labels(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({
"summary-labels": True,
"percentiles": False,
"summary": False,
"test-duration": False
})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
expected = (
"Request label stats:\n"
"+----------------------------------+--------+---------+--------+-----------+\n"
"| label | status | succ | avg_rt | error |\n"
"+----------------------------------+--------+---------+--------+-----------+\n"
"| http://192.168.1.1/anotherquery | FAIL | 0.00% | 0.001 | Forbidden |\n"
"| http://192.168.1.1/somequery | OK | 100.00% | 0.001 | |\n"
"| http://192.168.100.100/somequery | OK | 100.00% | 0.001 | |\n"
"+----------------------------------+--------+---------+--------+-----------+\n"
)
self.assertIn(expected, self.log_recorder.info_buff.getvalue())
def test_log_messages_failed_labels(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": True, "percentiles": False, "summary": False, "test-duration": False})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertIn("29656 failed samples: http://192.168.1.1/anotherquery\n", self.log_recorder.info_buff.getvalue())
def test_log_messages_samples_count(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": False, "percentiles": False, "summary": True,
"test-duration": False, "summary-labels": False})
self.sniff_log(obj.log)
obj.aggregated_second(self.__get_datapoint())
obj.startup()
obj.shutdown()
obj.post_process()
self.assertEqual("Samples count: 59314, 50.00% failures\n", self.log_recorder.info_buff.getvalue())
def test_log_messages_failed_labels(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": True, "percentiles": False, "summary": False,
"test-duration": False})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertIn("29656 failed samples: http://192.168.1.1/anotherquery\n", self.log_recorder.info_buff.getvalue())
def test_long_kpi(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"dump-xml": obj.engine.create_artifact("status", ".xml")})
datapoint = random_datapoint(time.time())
datapoint[datapoint.CUMULATIVE][""]["stdev_rt"] = long(0)
obj.aggregated_second(datapoint)
obj.startup()
obj.shutdown()
obj.post_process()
def test_log_messages_samples_count(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"failed-labels": False, "percentiles": False, "summary": True, "test-duration": False,
"summary-labels": False})
self.sniff_log(obj.log)
obj.aggregated_second(self.__get_datapoint())
obj.startup()
obj.shutdown()
obj.post_process()
self.assertEqual("Samples count: 59314, 50.00% failures\n", self.log_recorder.info_buff.getvalue())
def test_long_kpi(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict(
{"dump-xml": obj.engine.create_artifact("status", ".xml")})
datapoint = random_datapoint(time.time())
datapoint[datapoint.CUMULATIVE][""]["stdev_rt"] = long(0)
obj.aggregated_second(datapoint)
obj.startup()
obj.shutdown()
obj.post_process()
def test_dump(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({
"dump-xml": obj.engine.create_artifact("status", ".xml"),
"dump-csv": obj.engine.create_artifact("status", ".csv")
})
self.sniff_log(obj.log)
obj.aggregated_second(random_datapoint(time.time()))
obj.startup()
obj.shutdown()
obj.post_process()
self.assertIn("XML", self.log_recorder.info_buff.getvalue())
def test_dump(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({
"dump-xml": obj.engine.create_artifact("status", ".xml"),
"dump-csv": obj.engine.create_artifact("status", ".csv")
})
self.sniff_log(obj.log)
obj.aggregated_second(random_datapoint(time.time()))
obj.startup()
obj.shutdown()
obj.post_process()
self.assertIn("XML", self.log_recorder.info_buff.getvalue())
def test_log_messages_duration(self):
"""
Test duration report
:return:
"""
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.start_time -= 120005
obj.post_process()
self.assertEqual("Test duration: 1 day, 9:20:05\n", self.log_recorder.info_buff.getvalue())
def test_log_messages_duration(self):
"""
Test duration report
:return:
"""
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.start_time -= 120005
obj.post_process()
self.assertEqual("Test duration: 1 day, 9:20:05\n", self.log_recorder.info_buff.getvalue())
def test_func_report(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.aggregated_results(*self.__get_func_tree())
obj.post_process()
info_log = self.log_recorder.info_buff.getvalue()
warn_log = self.log_recorder.warn_buff.getvalue()
self.assertIn("Total: 3 tests", info_log)
self.assertIn("Test TestClass.case2 failed: something broke", warn_log)
self.assertIn("stacktrace2", warn_log)
self.assertIn("Test TestClass.case3 failed: something is badly broken", warn_log)
self.assertIn("stacktrace3", warn_log)
def test_func_report(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.aggregated_results(*self.__get_func_tree())
obj.post_process()
info_log = self.log_recorder.info_buff.getvalue()
warn_log = self.log_recorder.warn_buff.getvalue()
self.assertIn("Total: 3 tests", info_log)
self.assertIn("Test TestClass.case2 failed: something broke", warn_log)
self.assertIn("stacktrace2", warn_log)
self.assertIn("Test TestClass.case3 failed: something is badly broken", warn_log)
self.assertIn("stacktrace3", warn_log)
def test_func_report_all_no_stacktrace(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"report-tests": "all", "print-stacktrace": False})
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.aggregated_results(*self.__get_func_tree())
obj.post_process()
info_log = self.log_recorder.info_buff.getvalue()
self.assertIn("Total: 3 tests", info_log)
self.assertIn("Test TestClass.case1 - PASSED", info_log)
self.assertIn("Test TestClass.case2 - FAILED", info_log)
self.assertIn("Test TestClass.case3 - BROKEN", info_log)
self.assertNotIn("stacktrace2", info_log)
self.assertNotIn("stacktrace3", info_log)
def test_func_report_all_no_stacktrace(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"report-tests": "all", "print-stacktrace": False})
self.sniff_log(obj.log)
obj.prepare()
obj.startup()
obj.shutdown()
obj.aggregated_results(*self.__get_func_tree())
obj.post_process()
info_log = self.log_recorder.info_buff.getvalue()
self.assertIn("Total: 3 tests", info_log)
self.assertIn("Test TestClass.case1 - PASSED", info_log)
self.assertIn("Test TestClass.case2 - FAILED", info_log)
self.assertIn("Test TestClass.case3 - BROKEN", info_log)
self.assertNotIn("stacktrace2", info_log)
self.assertNotIn("stacktrace3", info_log)
def test_xml_report_test_duration(self):
obj = FinalStatus()
obj.engine = EngineEmul()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters = BetterDict.from_dict({
"dump-xml": xml_report,
})
obj.startup()
obj.aggregated_second(self.__get_datapoint(ts=90))
obj.aggregated_second(self.__get_datapoint(ts=100))
obj.shutdown()
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<TestDuration>10.0</TestDuration>', report_content)
def test_xml_report_test_duration(self):
obj = FinalStatus()
obj.engine = EngineEmul()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters = BetterDict.from_dict({
"dump-xml": xml_report,
})
obj.startup()
obj.aggregated_second(self.__get_datapoint(ts=90))
obj.aggregated_second(self.__get_datapoint(ts=100))
obj.shutdown()
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<TestDuration>10.0</TestDuration>', report_content)
def test_log_messages_percentiles(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
self.sniff_log(obj.log)
obj.parameters.merge({"failed-labels": False, "percentiles": True, "summary": False, "test-duration": False})
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
target_output = ("Average times: total 0.001, latency 0.000, connect 0.000\n"
"Percentile 0.0%: 0.000\n"
"Percentile 50.0%: 0.000\n"
"Percentile 90.0%: 0.001\n"
"Percentile 95.0%: 0.001\n"
"Percentile 99.0%: 0.003\n"
"Percentile 99.9%: 0.008\n"
"Percentile 100.0%: 0.081\n"
)
self.assertEqual(target_output, self.log_recorder.info_buff.getvalue())
def test_log_messages_summary_labels(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict.from_dict({"summary-labels": True, "percentiles": False, "summary": False,
"test-duration": False})
self.sniff_log(obj.log)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
expected = ("Request label stats:\n"
"+----------------------------------+--------+---------+--------+-----------+\n"
"| label | status | succ | avg_rt | error |\n"
"+----------------------------------+--------+---------+--------+-----------+\n"
"| http://192.168.1.1/anotherquery | FAIL | 0.00% | 0.001 | Forbidden |\n"
"| http://192.168.1.1/somequery | OK | 100.00% | 0.001 | |\n"
"| http://192.168.100.100/somequery | OK | 100.00% | 0.001 | |\n"
"+----------------------------------+--------+---------+--------+-----------+\n")
self.assertIn(expected, self.log_recorder.info_buff.getvalue())
def test_blazemeter_cloud_report_link(self):
obj = FinalStatus()
obj.engine = EngineEmul()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters = BetterDict.from_dict({
"dump-xml": xml_report,
})
prov = CloudProvisioning()
prov.results_url = "http://report/link"
obj.engine.provisioning = prov
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<ReportURL>http://report/link</ReportURL>', report_content)
def test_blazemeter_cloud_report_link(self):
obj = FinalStatus()
obj.engine = EngineEmul()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters = BetterDict.from_dict({
"dump-xml": xml_report,
})
prov = CloudProvisioning()
prov.results_url = "http://report/link"
obj.engine.provisioning = prov
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<ReportURL>http://report/link</ReportURL>', report_content)
def test_blazemeter_report_link(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters.merge({
"dump-xml": xml_report,
})
rep = BlazeMeterUploader()
rep.results_url = "http://report/link"
obj.engine.reporters.append(rep)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<ReportURL>http://report/link</ReportURL>', report_content)
def test_blazemeter_report_link(self):
obj = FinalStatus()
obj.engine = EngineEmul()
obj.parameters = BetterDict()
xml_report = obj.engine.create_artifact("status", ".xml")
obj.parameters.merge({
"dump-xml": xml_report,
})
rep = BlazeMeterUploader()
rep.results_url = "http://report/link"
obj.engine.reporters.append(rep)
obj.startup()
obj.shutdown()
obj.aggregated_second(self.__get_datapoint())
obj.post_process()
self.assertTrue(os.path.exists(xml_report))
with open(xml_report) as fds:
report_content = fds.read()
self.assertIn('<ReportURL>http://report/link</ReportURL>', report_content)
def test_csv_report_fieldname_order(self):
obj = FinalStatus()
obj.engine = EngineEmul()
csv_report = obj.engine.create_artifact("report", ".csv")
obj.parameters = BetterDict.from_dict({
"dump-csv": csv_report,
})
obj.startup()
obj.aggregated_second(self.__get_datapoint(ts=90))
obj.aggregated_second(self.__get_datapoint(ts=100))
obj.shutdown()
obj.post_process()
self.assertTrue(os.path.exists(csv_report))
with open(csv_report) as fds:
fieldnames = fds.readline().strip().split(",")
perc_fields = [float(name[5:]) for name in fieldnames if name.startswith('perc_')]
self.assertTrue(sorted(perc_fields) == perc_fields)
rc_fields = [float(name[3:]) for name in fieldnames if name.startswith('rc_')]
self.assertTrue(sorted(rc_fields) == rc_fields)
