def test_named_criteria(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": {"named criterion": "avg-rt of spaced label>10ms"}} obj.prepare() self.assertGreater(len(obj.criteria), 0) self.assertEquals(obj.criteria[0].message, "named criterion")
def test_named_criteria(self): obj = PassFailStatus() obj.parameters = { "criterias": { "named criteria": "avg-rt of spaced label>10ms" } } obj.prepare() self.assertGreater(len(obj.criterias), 0) self.assertEquals(obj.criterias[0].message, "named criteria")
def test_prepare2(self): obj = PassFailStatus() obj.parameters = {"criterias": ["avg-rt>10ms, continue as non-failed"]} obj.prepare() self.assertGreater(len(obj.criterias), 0) for n in range(0, 10): point = random_datapoint(n) obj.aggregated_second(point) obj.check() obj.post_process()
def test_xml_format_passfail(self): obj = JUnitXMLReporter() obj.engine = EngineEmul() obj.parameters = BetterDict() pass_fail1 = PassFailStatus() fc1_triggered = FailCriteria({'stop': True, 'label': 'Sample 1 Triggered', 'fail': True, 'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt'}) fc1_not_triggered = FailCriteria({'stop': True, 'label': 'Sample 1 Not Triggered', 'fail': True, 'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt'}) pass_fail2 = PassFailStatus() fc2_triggered = FailCriteria({'stop': True, 'label': 'Sample 2 Triggered', 'fail': True, 'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt'}) fc2_not_triggered = FailCriteria({'stop': True, 'label': 'Sample 2 Not Triggered', 'fail': True, 'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt'}) pass_fail1.criterias.append(fc1_triggered) pass_fail1.criterias.append(fc1_not_triggered) pass_fail2.criterias.append(fc2_triggered) pass_fail2.criterias.append(fc2_not_triggered) fc1_triggered.is_triggered = True fc2_triggered.is_triggered = True obj.engine.reporters.append(pass_fail1) obj.engine.reporters.append(pass_fail2) obj.engine.reporters.append(object()) path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml_passfail', dir=obj.engine.artifacts_dir) obj.parameters.merge({"filename": path_from_config, "data-source": "pass-fail"}) obj.prepare() obj.post_process() with open(obj.report_file_path, 'rb') as fds: f_contents = fds.read() xml_tree = etree.fromstring(f_contents) self.assertEqual('testsuite', xml_tree.tag) self.assertEqual(4, len(xml_tree.getchildren())) self.assertEqual('testcase', xml_tree.getchildren()[0].tag) self.assertEqual('error', xml_tree.getchildren()[0].getchildren()[0].tag) self.assertEqual('error', xml_tree.getchildren()[2].getchildren()[0].tag) self.assertListEqual(['2', 'taurus_junitxml_pass_fail', '0', '4'], xml_tree.values())
def test_widget(self): obj = PassFailStatus() obj.parameters = {"criterias": ["avg-rt>10ms for 2s, continue as failed"]} obj.prepare() obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 obj.aggregated_second(point) obj.check() start_time += 1 self.assertEqual(obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n")
def test_report_criteria_without_label(self): obj = JUnitXMLReporter() obj.engine = EngineEmul() obj.parameters = BetterDict() pass_fail = PassFailStatus() criteria = DataCriterion( { 'stop': True, 'fail': True, 'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt' }, pass_fail) pass_fail.criteria.append(criteria) criteria.is_triggered = True obj.engine.reporters.append(pass_fail) path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml_passfail', dir=obj.engine.artifacts_dir) obj.parameters.merge({ "filename": path_from_config, "data-source": "pass-fail" }) obj.prepare() obj.last_second = DataPoint(0) obj.post_process()
def test_passfail_crash(self): passfail = BetterDict() passfail.merge({ "module": "passfail", "criteria": [ "fail>10% within 5s", ] }) obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = passfail obj.engine.config.merge({ "services": [passfail], }) obj.prepare() self.assertTrue(all(isinstance(obj, dict) for obj in passfail["criteria"])) self.assertTrue(all(isinstance(obj, dict) for obj in passfail["criterias"]))
def test_prepare3(self): obj = PassFailStatus() obj.parameters = {"criterias": ["avg-rt>10ms for 3s, continue as failed"]} obj.prepare() self.assertGreater(len(obj.criterias), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1 obj.aggregated_second(point) obj.check() try: obj.post_process() self.fail() except AutomatedShutdown: pass
def test_within(self): obj = PassFailStatus() obj.parameters = {"criterias": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ]} obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) obj.aggregated_second(point) if _n % 2 == 0: obj.check() obj.check() start_time += 1
def test_prepare(self): obj = PassFailStatus() config = json.loads(open(__dir__() + "/../json/passfail.json").read()) obj.parameters = config['reporting'][0] obj.prepare() self.assertGreater(len(obj.criterias), 0) for n in range(0, 10): point = random_datapoint(n) logging.info("%s: %s", n, point) obj.aggregated_second(point) try: obj.check() except AutomatedShutdown: pass try: obj.post_process() except AutomatedShutdown: pass
def test_ashort_data(self): obj = PassFailStatus() obj.engine = EngineEmul() crit_cfg = DataCriterion.string_to_config("failures>0%, stop as failed") obj.criteria.append(DataCriterion(crit_cfg, obj)) point = DataPoint(0) point[DataPoint.CUMULATIVE] = {} point[DataPoint.CUMULATIVE][''] = {} point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16 point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16 obj.check() obj.shutdown() obj.aggregated_second(point) self.assertRaises(AutomatedShutdown, obj.post_process)
def test_passfail_crash(self): passfail = BetterDict() passfail.merge({ "module": "passfail", "criteria": [ "fail>10% within 5s", ] }) obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = passfail obj.engine.config.merge({ "services": [passfail], }) obj.prepare() self.assertTrue( all(isinstance(obj, dict) for obj in passfail["criteria"]))
def test_widget(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": ["avg-rt>10ms for 2s, continue as failed"] } obj.prepare() obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 obj.aggregated_second(point) obj.check() start_time += 1 self.assertEqual(obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n")
def test_prepare3(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": ["avg-rt>10ms for 3s, continue as failed"] } obj.prepare() self.assertGreater(len(obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1 obj.aggregated_second(point) obj.check() try: obj.post_process() self.fail() except AutomatedShutdown: pass
def test_prepare2(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": ["avg-rt>10ms, continue as non-failed"]} obj.prepare() self.assertGreater(len(obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) obj.aggregated_second(point) obj.check() obj.post_process()
def test_percentiles_track(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": ["p90>0ms"]} obj.prepare() self.assertGreater(len(obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) obj.aggregated_second(point) obj.check() obj.shutdown() try: obj.post_process() self.fail() except AutomatedShutdown: pass
def test_cumulative_criteria_post_process(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": [ "p90>0ms, continue as failed", "avg-rt>0ms, continue as failed", ]} obj.prepare() self.assertEquals(len(obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) obj.aggregated_second(point) obj.check() obj.shutdown() self.assertRaises(AutomatedShutdown, obj.post_process) for crit in obj.criteria: self.assertTrue(crit.is_triggered)
class TestPassFailStatus(BZTestCase): def setUp(self): super(TestPassFailStatus, self).setUp() self.obj = PassFailStatus() self.obj.engine = EngineEmul() def configure(self, params): self.obj.parameters.merge(params) def test_prepare(self): config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read()) self.configure(config['reporting'][0]) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) ROOT_LOGGER.info("%s: %s", n, point) self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: pass try: self.obj.post_process() except AutomatedShutdown: pass def test_prepare2(self): self.configure({"criteria": ["avg-rt>10ms, continue as non-failed"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.post_process() def test_prepare3(self): self.configure( {"criteria": ["avg-rt>10ms for 3s, continue as failed"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1 self.obj.aggregated_second(point) self.obj.check() self.assertRaises(AutomatedShutdown, self.obj.post_process) def test_widget(self): self.configure( {"criteria": ["avg-rt>10ms for 2s, continue as failed"]}) self.obj.prepare() self.obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time + _n) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 self.obj.aggregated_second(point) self.obj.check() self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n") def test_within(self): self.configure({ "criteria": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ] }) self.obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) self.obj.aggregated_second(point) if _n % 2 == 0: try: self.obj.check() except KeyboardInterrupt: pass try: self.obj.check() except KeyboardInterrupt: pass start_time += 1 def test_prepare_label_issue(self): # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ self.configure({"criteria": ["avg-rt of spaced label>10ms"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CUMULATIVE]['spaced label'] = point[ DataPoint.CUMULATIVE][''] point[DataPoint.CURRENT]['spaced label'] = point[ DataPoint.CURRENT][''] self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() try: self.obj.post_process() self.fail() except AutomatedShutdown: pass def test_named_criteria(self): self.configure( {"criteria": { "named criterion": "avg-rt of spaced label>10ms" }}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) self.assertEquals(self.obj.criteria[0].message, "named criterion") def test_stop_counting_criteria(self): self.configure( {"criteria": ["avg-rt>10ms for 2s, continue as failed"]}) self.obj.prepare() self.obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 self.obj.aggregated_second(point) self.obj.check() start_time += 1 self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n") for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 0.01 self.obj.aggregated_second(point) self.obj.check() start_time += 1 self.assertEqual(self.obj.widget.text_widget.text, "") def test_short_data(self): crit_cfg = DataCriterion.string_to_config( "failures>0%, stop as failed") self.obj.criteria.append(DataCriterion(crit_cfg, self.obj)) point = DataPoint(0) point[DataPoint.CUMULATIVE] = {} point[DataPoint.CUMULATIVE][''] = {} point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16 point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16 self.obj.check() self.obj.shutdown() self.obj.aggregated_second(point) self.assertRaises(AutomatedShutdown, self.obj.post_process) def test_passfail_crash(self): self.configure({ "module": "passfail", "criteria": ["fail>10% within 5s"] }) self.obj.engine.config.merge({ "services": [self.obj.parameters], }) self.obj.prepare() self.assertTrue( all( isinstance(obj, dict) for obj in self.obj.parameters["criteria"])) def test_percentiles_track(self): self.configure({"criteria": ["p90>0ms"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() try: self.obj.post_process() self.fail() except AutomatedShutdown: pass def test_cumulative_criteria_post_process(self): self.configure({ "criteria": [ "p90>0ms, continue as failed", "avg-rt>0ms, continue as failed", ] }) self.obj.prepare() self.assertEquals(len(self.obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() self.assertRaises(AutomatedShutdown, self.obj.post_process) for crit in self.obj.criteria: self.assertTrue(crit.is_triggered) def test_rc_within(self): self.configure({ "criteria": [ "rc413>10 within 3s, stop as successful", # this one to cover branch that caused bug "rc413>10 within 10s, stop as failed", ] }) self.obj.prepare() self.assertEquals(len(self.obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['413'] = 3 self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: break self.assertLess(n, 3) self.obj.shutdown() self.obj.post_process() self.assertFalse(self.obj.criteria[0].is_triggered) self.assertTrue(self.obj.criteria[1].is_triggered) def test_rc_over1(self): self.configure({"criteria": [ "rc200<8 over 5s", ]}) self.obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 3 self.obj.aggregated_second(point) self.obj.check() self.assertFalse(self.obj.criteria[0].is_triggered) self.obj.shutdown() self.obj.post_process() def test_rc_over2(self): self.configure({"criteria": [ "rc200>8 over 3s", ]}) self.obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 5 self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: break self.assertLess(n, 3) self.assertTrue(self.obj.criteria[0].is_triggered) self.obj.shutdown() self.obj.post_process()
def setUp(self): super(TestPassFailStatus, self).setUp() self.obj = PassFailStatus() self.obj.engine = EngineEmul()
def test_rc_over2(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": [ "rc200>8 over 3s", ] } obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 5 # logging.debug("Datapoint %s: %s", n, point) obj.aggregated_second(point) try: obj.check() except AutomatedShutdown: break self.assertLess(n, 3) self.assertTrue(obj.criteria[0].is_triggered) obj.shutdown() obj.post_process()
class TestPassFailStatus(BZTestCase): def setUp(self): super(TestPassFailStatus, self).setUp() self.obj = PassFailStatus() self.obj.engine = EngineEmul() def configure(self, params): self.obj.parameters.merge(params) def test_prepare(self): config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read()) self.configure(config['reporting'][0]) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) logging.info("%s: %s", n, point) self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: pass try: self.obj.post_process() except AutomatedShutdown: pass def test_prepare2(self): self.configure({"criteria": ["avg-rt>10ms, continue as non-failed"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.post_process() def test_prepare3(self): self.configure({"criteria": ["avg-rt>10ms for 3s, continue as failed"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1 self.obj.aggregated_second(point) self.obj.check() self.assertRaises(AutomatedShutdown, self.obj.post_process) def test_widget(self): self.configure({"criteria": ["avg-rt>10ms for 2s, continue as failed"]}) self.obj.prepare() self.obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time + _n) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 self.obj.aggregated_second(point) self.obj.check() self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n") def test_within(self): self.configure({ "criteria": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ]}) self.obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) self.obj.aggregated_second(point) if _n % 2 == 0: try: self.obj.check() except KeyboardInterrupt: pass try: self.obj.check() except KeyboardInterrupt: pass start_time += 1 def test_prepare_label_issue(self): # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ self.configure({"criteria": ["avg-rt of spaced label>10ms"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CUMULATIVE]['spaced label'] = point[DataPoint.CUMULATIVE][''] point[DataPoint.CURRENT]['spaced label'] = point[DataPoint.CURRENT][''] self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() try: self.obj.post_process() self.fail() except AutomatedShutdown: pass def test_named_criteria(self): self.configure({"criteria": {"named criterion": "avg-rt of spaced label>10ms"}}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) self.assertEquals(self.obj.criteria[0].message, "named criterion") def test_stop_counting_criteria(self): self.configure({"criteria": ["avg-rt>10ms for 2s, continue as failed"]}) self.obj.prepare() self.obj.get_widget() start_time = time.time() for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 1.0 self.obj.aggregated_second(point) self.obj.check() start_time += 1 self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n") for _n in range(0, 10): point = random_datapoint(start_time) point[DataPoint.CURRENT]['']["avg_rt"] = 0.01 self.obj.aggregated_second(point) self.obj.check() start_time += 1 self.assertEqual(self.obj.widget.text_widget.text, "") def test_short_data(self): crit_cfg = DataCriterion.string_to_config("failures>0%, stop as failed") self.obj.criteria.append(DataCriterion(crit_cfg, self.obj)) point = DataPoint(0) point[DataPoint.CUMULATIVE] = {} point[DataPoint.CUMULATIVE][''] = {} point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16 point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16 self.obj.check() self.obj.shutdown() self.obj.aggregated_second(point) self.assertRaises(AutomatedShutdown, self.obj.post_process) def test_passfail_crash(self): self.configure({ "module": "passfail", "criteria": [ "fail>10% within 5s"] }) self.obj.engine.config.merge({ "services": [self.obj.parameters], }) self.obj.prepare() self.assertTrue(all(isinstance(obj, dict) for obj in self.obj.parameters["criteria"])) def test_percentiles_track(self): self.configure({"criteria": ["p90>0ms"]}) self.obj.prepare() self.assertGreater(len(self.obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() try: self.obj.post_process() self.fail() except AutomatedShutdown: pass def test_cumulative_criteria_post_process(self): self.configure({"criteria": [ "p90>0ms, continue as failed", "avg-rt>0ms, continue as failed", ]}) self.obj.prepare() self.assertEquals(len(self.obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) self.obj.aggregated_second(point) self.obj.check() self.obj.shutdown() self.assertRaises(AutomatedShutdown, self.obj.post_process) for crit in self.obj.criteria: self.assertTrue(crit.is_triggered) def test_rc_within(self): self.configure({"criteria": [ "rc413>10 within 3s, stop as successful", # this one to cover branch that caused bug "rc413>10 within 10s, stop as failed", ]}) self.obj.prepare() self.assertEquals(len(self.obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['413'] = 3 self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: break self.assertLess(n, 3) self.obj.shutdown() self.obj.post_process() self.assertFalse(self.obj.criteria[0].is_triggered) self.assertTrue(self.obj.criteria[1].is_triggered) def test_rc_over1(self): self.configure({"criteria": [ "rc200<8 over 5s", ]}) self.obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 3 self.obj.aggregated_second(point) self.obj.check() self.assertFalse(self.obj.criteria[0].is_triggered) self.obj.shutdown() self.obj.post_process() def test_rc_over2(self): self.configure({"criteria": [ "rc200>8 over 3s", ]}) self.obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 5 # logging.debug("Datapoint %s: %s", n, point) self.obj.aggregated_second(point) try: self.obj.check() except AutomatedShutdown: break self.assertLess(n, 3) self.assertTrue(self.obj.criteria[0].is_triggered) self.obj.shutdown() self.obj.post_process()
def test_cumulative_criteria_post_process(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": [ "p90>0ms, continue as failed", "avg-rt>0ms, continue as failed", ] } obj.prepare() self.assertEquals(len(obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) obj.aggregated_second(point) obj.check() obj.shutdown() self.assertRaises(AutomatedShutdown, obj.post_process) for crit in obj.criteria: self.assertTrue(crit.is_triggered)
def test_within(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ]} obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) obj.aggregated_second(point) if _n % 2 == 0: try: obj.check() except KeyboardInterrupt: pass try: obj.check() except KeyboardInterrupt: pass start_time += 1
def test_within(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ] } obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) obj.aggregated_second(point) if _n % 2 == 0: try: obj.check() except KeyboardInterrupt: pass try: obj.check() except KeyboardInterrupt: pass start_time += 1
def test_xml_format_passfail(self): obj = JUnitXMLReporter() obj.engine = EngineEmul() obj.parameters = BetterDict() obj.engine.provisioning = CloudProvisioning() obj.engine.provisioning.results_url = "http://test/report/123" pass_fail1 = PassFailStatus() fc1_triggered = DataCriterion( { 'stop': True, 'label': 'Sample 1 Triggered', 'fail': True, 'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt' }, pass_fail1) fc1_not_triggered = DataCriterion( { 'stop': True, 'label': 'Sample 1 Not Triggered', 'fail': True, 'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt' }, pass_fail1) pass_fail2 = PassFailStatus() fc2_triggered = DataCriterion( { 'stop': True, 'label': 'Sample 2 Triggered', 'fail': True, 'timeframe': -1, 'threshold': '150ms', 'condition': '<=', 'subject': 'avg-rt' }, pass_fail1) fc2_not_triggered = DataCriterion( { 'stop': True, 'label': 'Sample 2 Not Triggered', 'fail': True, 'timeframe': -1, 'threshold': '300ms', 'condition': '=', 'subject': 'avg-rt' }, pass_fail1) pass_fail1.criteria.append(fc1_triggered) pass_fail1.criteria.append(fc1_not_triggered) pass_fail2.criteria.append(fc2_triggered) pass_fail2.criteria.append(fc2_not_triggered) fc1_triggered.is_triggered = True fc2_triggered.is_triggered = True obj.engine.reporters.append(pass_fail1) obj.engine.reporters.append(pass_fail2) obj.engine.reporters.append(BlazeMeterUploader()) path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml_passfail', dir=obj.engine.artifacts_dir) obj.parameters.merge({ "filename": path_from_config, "data-source": "pass-fail" }) obj.prepare() obj.last_second = DataPoint(0) obj.post_process() with open(obj.report_file_path, 'rb') as fds: f_contents = fds.read() logging.info("File: %s", f_contents) xml_tree = etree.fromstring(f_contents) self.assertEqual('testsuites', xml_tree.tag) suite = xml_tree.getchildren()[0] self.assertEqual('testsuite', suite.tag) test_cases = suite.getchildren() self.assertEqual(4, len(test_cases)) self.assertEqual('testcase', test_cases[0].tag) self.assertEqual('error', test_cases[0].getchildren()[1].tag) self.assertEqual('error', test_cases[2].getchildren()[1].tag) sys_out = test_cases[0].getchildren()[0] self.assertEqual('system-out', sys_out.tag) self.assertIn('BlazeMeter report link: http://test/report/123', sys_out.text)
def test_prepare_label_issue(self): # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ obj = PassFailStatus() obj.parameters = {"criterias": ["avg-rt of spaced label>10ms"]} obj.prepare() self.assertGreater(len(obj.criterias), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CUMULATIVE]['spaced label'] = point[DataPoint.CUMULATIVE][''] point[DataPoint.CURRENT]['spaced label'] = point[DataPoint.CURRENT][''] obj.aggregated_second(point) obj.check() obj.shutdown() try: obj.post_process() self.fail() except AutomatedShutdown: pass
def test_prepare_label_issue(self): # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = {"criteria": ["avg-rt of spaced label>10ms"]} obj.prepare() self.assertGreater(len(obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) point[DataPoint.CUMULATIVE]['spaced label'] = point[ DataPoint.CUMULATIVE][''] point[DataPoint.CURRENT]['spaced label'] = point[ DataPoint.CURRENT][''] obj.aggregated_second(point) obj.check() obj.shutdown() try: obj.post_process() self.fail() except AutomatedShutdown: pass
def test_prepare(self): obj = PassFailStatus() obj.engine = EngineEmul() config = json.loads(open(__dir__() + "/../json/passfail.json").read()) obj.parameters = config['reporting'][0] obj.prepare() self.assertGreater(len(obj.criteria), 0) for n in range(0, 10): point = random_datapoint(n) logging.info("%s: %s", n, point) obj.aggregated_second(point) try: obj.check() except AutomatedShutdown: pass try: obj.post_process() except AutomatedShutdown: pass
def test_rc_within(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": [ "rc413>10 within 3s, stop as successful", # this one to cover branch that caused bug "rc413>10 within 10s, stop as failed", ] } obj.prepare() self.assertEquals(len(obj.criteria), 2) for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['413'] = 3 obj.aggregated_second(point) try: obj.check() except AutomatedShutdown: break self.assertLess(n, 3) obj.shutdown() obj.post_process() self.assertFalse(obj.criteria[0].is_triggered) self.assertTrue(obj.criteria[1].is_triggered)
def test_rc_over1(self): obj = PassFailStatus() obj.engine = EngineEmul() obj.parameters = { "criteria": [ "rc200<8 over 5s", ] } obj.prepare() for n in range(0, 10): point = random_datapoint(n) rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES] rcs['200'] = 3 obj.aggregated_second(point) obj.check() self.assertFalse(obj.criteria[0].is_triggered) obj.shutdown() obj.post_process()
def test_within(self): obj = PassFailStatus() obj.parameters = { "criterias": [ "fail>10% within 5s", "fail>1000 within 5s", "avg-rt>100ms within 10s", ] } obj.prepare() start_time = time.time() for _n in range(0, 20): point = random_datapoint(start_time) obj.aggregated_second(point) if _n % 2 == 0: obj.check() obj.check() start_time += 1