Ejemplo n.º 1
0
 def test_named_criteria(self):
     obj = PassFailStatus()
     obj.engine = EngineEmul()
     obj.parameters = {"criteria": {"named criterion": "avg-rt of spaced label>10ms"}}
     obj.prepare()
     self.assertGreater(len(obj.criteria), 0)
     self.assertEquals(obj.criteria[0].message, "named criterion")
Ejemplo n.º 2
0
 def test_named_criteria(self):
     obj = PassFailStatus()
     obj.parameters = {
         "criterias": {
             "named criteria": "avg-rt of spaced label>10ms"
         }
     }
     obj.prepare()
     self.assertGreater(len(obj.criterias), 0)
     self.assertEquals(obj.criterias[0].message, "named criteria")
Ejemplo n.º 3
0
    def test_prepare2(self):
        obj = PassFailStatus()
        obj.parameters = {"criterias": ["avg-rt>10ms, continue as non-failed"]}
        obj.prepare()
        self.assertGreater(len(obj.criterias), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.post_process()
Ejemplo n.º 4
0
    def test_xml_format_passfail(self):

        obj = JUnitXMLReporter()
        obj.engine = EngineEmul()
        obj.parameters = BetterDict()

        pass_fail1 = PassFailStatus()

        fc1_triggered = FailCriteria({'stop': True, 'label': 'Sample 1 Triggered', 'fail': True,
                                      'timeframe': -1, 'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt'})

        fc1_not_triggered = FailCriteria({'stop': True, 'label': 'Sample 1 Not Triggered', 'fail': True,
                                          'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt'})

        pass_fail2 = PassFailStatus()

        fc2_triggered = FailCriteria({'stop': True, 'label': 'Sample 2 Triggered', 'fail': True, 'timeframe': -1,
                                      'threshold': '150ms', 'condition': '<', 'subject': 'avg-rt'})

        fc2_not_triggered = FailCriteria({'stop': True, 'label': 'Sample 2 Not Triggered', 'fail': True,
                                          'timeframe': -1, 'threshold': '300ms', 'condition': '>', 'subject': 'avg-rt'})

        pass_fail1.criterias.append(fc1_triggered)
        pass_fail1.criterias.append(fc1_not_triggered)
        pass_fail2.criterias.append(fc2_triggered)
        pass_fail2.criterias.append(fc2_not_triggered)

        fc1_triggered.is_triggered = True
        fc2_triggered.is_triggered = True

        obj.engine.reporters.append(pass_fail1)
        obj.engine.reporters.append(pass_fail2)
        obj.engine.reporters.append(object())

        path_from_config = tempfile.mktemp(suffix='.xml', prefix='junit-xml_passfail', dir=obj.engine.artifacts_dir)

        obj.parameters.merge({"filename": path_from_config, "data-source": "pass-fail"})
        obj.prepare()
        obj.post_process()

        with open(obj.report_file_path, 'rb') as fds:
            f_contents = fds.read()


        xml_tree = etree.fromstring(f_contents)
        self.assertEqual('testsuite', xml_tree.tag)
        self.assertEqual(4, len(xml_tree.getchildren()))
        self.assertEqual('testcase', xml_tree.getchildren()[0].tag)
        self.assertEqual('error', xml_tree.getchildren()[0].getchildren()[0].tag)
        self.assertEqual('error', xml_tree.getchildren()[2].getchildren()[0].tag)
        self.assertListEqual(['2', 'taurus_junitxml_pass_fail', '0', '4'], xml_tree.values())
Ejemplo n.º 5
0
    def test_widget(self):
        obj = PassFailStatus()
        obj.parameters = {"criterias": ["avg-rt>10ms for 2s, continue as failed"]}
        obj.prepare()
        obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            obj.aggregated_second(point)
            obj.check()
            start_time += 1

        self.assertEqual(obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n")
Ejemplo n.º 6
0
    def test_report_criteria_without_label(self):
        obj = JUnitXMLReporter()
        obj.engine = EngineEmul()
        obj.parameters = BetterDict()

        pass_fail = PassFailStatus()

        criteria = DataCriterion(
            {
                'stop': True,
                'fail': True,
                'timeframe': -1,
                'threshold': '150ms',
                'condition': '<',
                'subject': 'avg-rt'
            }, pass_fail)
        pass_fail.criteria.append(criteria)
        criteria.is_triggered = True

        obj.engine.reporters.append(pass_fail)

        path_from_config = tempfile.mktemp(suffix='.xml',
                                           prefix='junit-xml_passfail',
                                           dir=obj.engine.artifacts_dir)
        obj.parameters.merge({
            "filename": path_from_config,
            "data-source": "pass-fail"
        })
        obj.prepare()
        obj.last_second = DataPoint(0)
        obj.post_process()
Ejemplo n.º 7
0
 def test_passfail_crash(self):
     passfail = BetterDict()
     passfail.merge({
         "module": "passfail",
         "criteria": [
             "fail>10% within 5s",
         ]
     })
     obj = PassFailStatus()
     obj.engine = EngineEmul()
     obj.parameters = passfail
     obj.engine.config.merge({
         "services": [passfail],
     })
     obj.prepare()
     self.assertTrue(all(isinstance(obj, dict) for obj in passfail["criteria"]))
     self.assertTrue(all(isinstance(obj, dict) for obj in passfail["criterias"]))
Ejemplo n.º 8
0
    def test_prepare3(self):
        obj = PassFailStatus()
        obj.parameters = {"criterias": ["avg-rt>10ms for 3s, continue as failed"]}
        obj.prepare()
        self.assertGreater(len(obj.criterias), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1
            obj.aggregated_second(point)
            obj.check()

        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 9
0
    def test_within(self):
        obj = PassFailStatus()
        obj.parameters = {"criterias": [
            "fail>10% within 5s",
            "fail>1000 within 5s",
            "avg-rt>100ms within 10s",
        ]}
        obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            obj.aggregated_second(point)
            if _n % 2 == 0:
                obj.check()
            obj.check()
            start_time += 1
Ejemplo n.º 10
0
    def test_prepare(self):
        obj = PassFailStatus()
        config = json.loads(open(__dir__() + "/../json/passfail.json").read())
        obj.parameters = config['reporting'][0]
        obj.prepare()
        self.assertGreater(len(obj.criterias), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            logging.info("%s: %s", n, point)
            obj.aggregated_second(point)
            try:
                obj.check()
            except AutomatedShutdown:
                pass

        try:
            obj.post_process()
        except AutomatedShutdown:
            pass
Ejemplo n.º 11
0
    def test_ashort_data(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()

        crit_cfg = DataCriterion.string_to_config("failures>0%, stop as failed")
        obj.criteria.append(DataCriterion(crit_cfg, obj))

        point = DataPoint(0)
        point[DataPoint.CUMULATIVE] = {}
        point[DataPoint.CUMULATIVE][''] = {}
        point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16
        point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16

        obj.check()
        obj.shutdown()
        obj.aggregated_second(point)
        self.assertRaises(AutomatedShutdown, obj.post_process)
Ejemplo n.º 12
0
 def test_passfail_crash(self):
     passfail = BetterDict()
     passfail.merge({
         "module": "passfail",
         "criteria": [
             "fail>10% within 5s",
         ]
     })
     obj = PassFailStatus()
     obj.engine = EngineEmul()
     obj.parameters = passfail
     obj.engine.config.merge({
         "services": [passfail],
     })
     obj.prepare()
     self.assertTrue(
         all(isinstance(obj, dict) for obj in passfail["criteria"]))
Ejemplo n.º 13
0
    def test_widget(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": ["avg-rt>10ms for 2s, continue as failed"]
        }
        obj.prepare()
        obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            obj.aggregated_second(point)
            obj.check()
            start_time += 1

        self.assertEqual(obj.widget.text_widget.text,
                         "Failed: avg-rt>10ms for 10 sec\n")
Ejemplo n.º 14
0
    def test_prepare3(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": ["avg-rt>10ms for 3s, continue as failed"]
        }
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1
            obj.aggregated_second(point)
            obj.check()

        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 15
0
    def test_prepare2(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": ["avg-rt>10ms, continue as non-failed"]}
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.post_process()
Ejemplo n.º 16
0
    def test_percentiles_track(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": ["p90>0ms"]}
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()
        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 17
0
    def test_cumulative_criteria_post_process(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": [
            "p90>0ms, continue as failed",
            "avg-rt>0ms, continue as failed",
        ]}
        obj.prepare()
        self.assertEquals(len(obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()
        self.assertRaises(AutomatedShutdown, obj.post_process)
        for crit in obj.criteria:
            self.assertTrue(crit.is_triggered)
Ejemplo n.º 18
0
class TestPassFailStatus(BZTestCase):
    def setUp(self):
        super(TestPassFailStatus, self).setUp()
        self.obj = PassFailStatus()
        self.obj.engine = EngineEmul()

    def configure(self, params):
        self.obj.parameters.merge(params)

    def test_prepare(self):
        config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read())
        self.configure(config['reporting'][0])
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            ROOT_LOGGER.info("%s: %s", n, point)
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                pass

        try:
            self.obj.post_process()
        except AutomatedShutdown:
            pass

    def test_prepare2(self):
        self.configure({"criteria": ["avg-rt>10ms, continue as non-failed"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.post_process()

    def test_prepare3(self):
        self.configure(
            {"criteria": ["avg-rt>10ms for 3s, continue as failed"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1
            self.obj.aggregated_second(point)
            self.obj.check()

        self.assertRaises(AutomatedShutdown, self.obj.post_process)

    def test_widget(self):
        self.configure(
            {"criteria": ["avg-rt>10ms for 2s, continue as failed"]})
        self.obj.prepare()
        self.obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time + _n)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            self.obj.aggregated_second(point)
            self.obj.check()

        self.assertEqual(self.obj.widget.text_widget.text,
                         "Failed: avg-rt>10ms for 10 sec\n")

    def test_within(self):
        self.configure({
            "criteria": [
                "fail>10% within 5s",
                "fail>1000 within 5s",
                "avg-rt>100ms within 10s",
            ]
        })
        self.obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            self.obj.aggregated_second(point)
            if _n % 2 == 0:
                try:
                    self.obj.check()
                except KeyboardInterrupt:
                    pass

            try:
                self.obj.check()
            except KeyboardInterrupt:
                pass
            start_time += 1

    def test_prepare_label_issue(self):
        # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ
        self.configure({"criteria": ["avg-rt of spaced label>10ms"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CUMULATIVE]['spaced label'] = point[
                DataPoint.CUMULATIVE]['']
            point[DataPoint.CURRENT]['spaced label'] = point[
                DataPoint.CURRENT]['']
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()

        try:
            self.obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass

    def test_named_criteria(self):
        self.configure(
            {"criteria": {
                "named criterion": "avg-rt of spaced label>10ms"
            }})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)
        self.assertEquals(self.obj.criteria[0].message, "named criterion")

    def test_stop_counting_criteria(self):
        self.configure(
            {"criteria": ["avg-rt>10ms for 2s, continue as failed"]})
        self.obj.prepare()
        self.obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            self.obj.aggregated_second(point)
            self.obj.check()
            start_time += 1

        self.assertEqual(self.obj.widget.text_widget.text,
                         "Failed: avg-rt>10ms for 10 sec\n")

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 0.01
            self.obj.aggregated_second(point)
            self.obj.check()
            start_time += 1

        self.assertEqual(self.obj.widget.text_widget.text, "")

    def test_short_data(self):
        crit_cfg = DataCriterion.string_to_config(
            "failures>0%, stop as failed")
        self.obj.criteria.append(DataCriterion(crit_cfg, self.obj))

        point = DataPoint(0)
        point[DataPoint.CUMULATIVE] = {}
        point[DataPoint.CUMULATIVE][''] = {}
        point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16
        point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16

        self.obj.check()
        self.obj.shutdown()
        self.obj.aggregated_second(point)
        self.assertRaises(AutomatedShutdown, self.obj.post_process)

    def test_passfail_crash(self):
        self.configure({
            "module": "passfail",
            "criteria": ["fail>10% within 5s"]
        })

        self.obj.engine.config.merge({
            "services": [self.obj.parameters],
        })

        self.obj.prepare()
        self.assertTrue(
            all(
                isinstance(obj, dict)
                for obj in self.obj.parameters["criteria"]))

    def test_percentiles_track(self):
        self.configure({"criteria": ["p90>0ms"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()
        try:
            self.obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass

    def test_cumulative_criteria_post_process(self):
        self.configure({
            "criteria": [
                "p90>0ms, continue as failed",
                "avg-rt>0ms, continue as failed",
            ]
        })
        self.obj.prepare()
        self.assertEquals(len(self.obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()
        self.assertRaises(AutomatedShutdown, self.obj.post_process)
        for crit in self.obj.criteria:
            self.assertTrue(crit.is_triggered)

    def test_rc_within(self):
        self.configure({
            "criteria": [
                "rc413>10 within 3s, stop as successful",  # this one to cover branch that caused bug
                "rc413>10 within 10s, stop as failed",
            ]
        })
        self.obj.prepare()
        self.assertEquals(len(self.obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['413'] = 3
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        self.obj.shutdown()
        self.obj.post_process()
        self.assertFalse(self.obj.criteria[0].is_triggered)
        self.assertTrue(self.obj.criteria[1].is_triggered)

    def test_rc_over1(self):
        self.configure({"criteria": [
            "rc200<8 over 5s",
        ]})

        self.obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 3
            self.obj.aggregated_second(point)
            self.obj.check()
            self.assertFalse(self.obj.criteria[0].is_triggered)

        self.obj.shutdown()
        self.obj.post_process()

    def test_rc_over2(self):
        self.configure({"criteria": [
            "rc200>8 over 3s",
        ]})
        self.obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 5
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        self.assertTrue(self.obj.criteria[0].is_triggered)
        self.obj.shutdown()
        self.obj.post_process()
Ejemplo n.º 19
0
 def setUp(self):
     super(TestPassFailStatus, self).setUp()
     self.obj = PassFailStatus()
     self.obj.engine = EngineEmul()
Ejemplo n.º 20
0
    def test_rc_over2(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": [
                "rc200>8 over 3s",
            ]
        }
        obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 5
            # logging.debug("Datapoint %s: %s", n, point)
            obj.aggregated_second(point)
            try:
                obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        self.assertTrue(obj.criteria[0].is_triggered)
        obj.shutdown()
        obj.post_process()
Ejemplo n.º 21
0
class TestPassFailStatus(BZTestCase):
    def setUp(self):
        super(TestPassFailStatus, self).setUp()
        self.obj = PassFailStatus()
        self.obj.engine = EngineEmul()

    def configure(self, params):
        self.obj.parameters.merge(params)

    def test_prepare(self):
        config = json.loads(open(RESOURCES_DIR + "json/passfail.json").read())
        self.configure(config['reporting'][0])
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            logging.info("%s: %s", n, point)
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                pass

        try:
            self.obj.post_process()
        except AutomatedShutdown:
            pass

    def test_prepare2(self):
        self.configure({"criteria": ["avg-rt>10ms, continue as non-failed"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.post_process()

    def test_prepare3(self):
        self.configure({"criteria": ["avg-rt>10ms for 3s, continue as failed"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CURRENT][''][KPISet.AVG_RESP_TIME] = 1
            self.obj.aggregated_second(point)
            self.obj.check()

        self.assertRaises(AutomatedShutdown, self.obj.post_process)

    def test_widget(self):
        self.configure({"criteria": ["avg-rt>10ms for 2s, continue as failed"]})
        self.obj.prepare()
        self.obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time + _n)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            self.obj.aggregated_second(point)
            self.obj.check()

        self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n")

    def test_within(self):
        self.configure({
            "criteria": [
                "fail>10% within 5s",
                "fail>1000 within 5s",
                "avg-rt>100ms within 10s",
            ]})
        self.obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            self.obj.aggregated_second(point)
            if _n % 2 == 0:
                try:
                    self.obj.check()
                except KeyboardInterrupt:
                    pass

            try:
                self.obj.check()
            except KeyboardInterrupt:
                pass
            start_time += 1

    def test_prepare_label_issue(self):
        # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ
        self.configure({"criteria": ["avg-rt of spaced label>10ms"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CUMULATIVE]['spaced label'] = point[DataPoint.CUMULATIVE]['']
            point[DataPoint.CURRENT]['spaced label'] = point[DataPoint.CURRENT]['']
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()

        try:
            self.obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass

    def test_named_criteria(self):
        self.configure({"criteria": {"named criterion": "avg-rt of spaced label>10ms"}})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)
        self.assertEquals(self.obj.criteria[0].message, "named criterion")

    def test_stop_counting_criteria(self):
        self.configure({"criteria": ["avg-rt>10ms for 2s, continue as failed"]})
        self.obj.prepare()
        self.obj.get_widget()
        start_time = time.time()

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 1.0
            self.obj.aggregated_second(point)
            self.obj.check()
            start_time += 1

        self.assertEqual(self.obj.widget.text_widget.text, "Failed: avg-rt>10ms for 10 sec\n")

        for _n in range(0, 10):
            point = random_datapoint(start_time)
            point[DataPoint.CURRENT]['']["avg_rt"] = 0.01
            self.obj.aggregated_second(point)
            self.obj.check()
            start_time += 1

        self.assertEqual(self.obj.widget.text_widget.text, "")

    def test_short_data(self):
        crit_cfg = DataCriterion.string_to_config("failures>0%, stop as failed")
        self.obj.criteria.append(DataCriterion(crit_cfg, self.obj))

        point = DataPoint(0)
        point[DataPoint.CUMULATIVE] = {}
        point[DataPoint.CUMULATIVE][''] = {}
        point[DataPoint.CUMULATIVE][''][KPISet.FAILURES] = 100 * 16
        point[DataPoint.CUMULATIVE][''][KPISet.SAMPLE_COUNT] = 100 * 16

        self.obj.check()
        self.obj.shutdown()
        self.obj.aggregated_second(point)
        self.assertRaises(AutomatedShutdown, self.obj.post_process)

    def test_passfail_crash(self):
        self.configure({
            "module": "passfail",
            "criteria": [
                "fail>10% within 5s"]
        })

        self.obj.engine.config.merge({
            "services": [self.obj.parameters],
        })

        self.obj.prepare()
        self.assertTrue(all(isinstance(obj, dict) for obj in self.obj.parameters["criteria"]))

    def test_percentiles_track(self):
        self.configure({"criteria": ["p90>0ms"]})
        self.obj.prepare()
        self.assertGreater(len(self.obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()
        try:
            self.obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass

    def test_cumulative_criteria_post_process(self):
        self.configure({"criteria": [
            "p90>0ms, continue as failed",
            "avg-rt>0ms, continue as failed",
        ]})
        self.obj.prepare()
        self.assertEquals(len(self.obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            self.obj.aggregated_second(point)
            self.obj.check()

        self.obj.shutdown()
        self.assertRaises(AutomatedShutdown, self.obj.post_process)
        for crit in self.obj.criteria:
            self.assertTrue(crit.is_triggered)

    def test_rc_within(self):
        self.configure({"criteria": [
            "rc413>10 within 3s, stop as successful",  # this one to cover branch that caused bug
            "rc413>10 within 10s, stop as failed",
        ]})
        self.obj.prepare()
        self.assertEquals(len(self.obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['413'] = 3
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        self.obj.shutdown()
        self.obj.post_process()
        self.assertFalse(self.obj.criteria[0].is_triggered)
        self.assertTrue(self.obj.criteria[1].is_triggered)

    def test_rc_over1(self):
        self.configure({"criteria": [
            "rc200<8 over 5s",
        ]})

        self.obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 3
            self.obj.aggregated_second(point)
            self.obj.check()
            self.assertFalse(self.obj.criteria[0].is_triggered)

        self.obj.shutdown()
        self.obj.post_process()

    def test_rc_over2(self):
        self.configure({"criteria": [
            "rc200>8 over 3s",
        ]})
        self.obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 5
            # logging.debug("Datapoint %s: %s", n, point)
            self.obj.aggregated_second(point)
            try:
                self.obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        self.assertTrue(self.obj.criteria[0].is_triggered)
        self.obj.shutdown()
        self.obj.post_process()
Ejemplo n.º 22
0
 def setUp(self):
     super(TestPassFailStatus, self).setUp()
     self.obj = PassFailStatus()
     self.obj.engine = EngineEmul()
Ejemplo n.º 23
0
    def test_cumulative_criteria_post_process(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": [
                "p90>0ms, continue as failed",
                "avg-rt>0ms, continue as failed",
            ]
        }
        obj.prepare()
        self.assertEquals(len(obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()
        self.assertRaises(AutomatedShutdown, obj.post_process)
        for crit in obj.criteria:
            self.assertTrue(crit.is_triggered)
Ejemplo n.º 24
0
    def test_within(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": [
            "fail>10% within 5s",
            "fail>1000 within 5s",
            "avg-rt>100ms within 10s",
        ]}
        obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            obj.aggregated_second(point)
            if _n % 2 == 0:
                try:
                    obj.check()
                except KeyboardInterrupt:
                    pass

            try:
                obj.check()
            except KeyboardInterrupt:
                pass
            start_time += 1
Ejemplo n.º 25
0
    def test_within(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": [
                "fail>10% within 5s",
                "fail>1000 within 5s",
                "avg-rt>100ms within 10s",
            ]
        }
        obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            obj.aggregated_second(point)
            if _n % 2 == 0:
                try:
                    obj.check()
                except KeyboardInterrupt:
                    pass

            try:
                obj.check()
            except KeyboardInterrupt:
                pass
            start_time += 1
Ejemplo n.º 26
0
    def test_xml_format_passfail(self):
        obj = JUnitXMLReporter()
        obj.engine = EngineEmul()
        obj.parameters = BetterDict()
        obj.engine.provisioning = CloudProvisioning()
        obj.engine.provisioning.results_url = "http://test/report/123"

        pass_fail1 = PassFailStatus()

        fc1_triggered = DataCriterion(
            {
                'stop': True,
                'label': 'Sample 1 Triggered',
                'fail': True,
                'timeframe': -1,
                'threshold': '150ms',
                'condition': '<',
                'subject': 'avg-rt'
            }, pass_fail1)

        fc1_not_triggered = DataCriterion(
            {
                'stop': True,
                'label': 'Sample 1 Not Triggered',
                'fail': True,
                'timeframe': -1,
                'threshold': '300ms',
                'condition': '>',
                'subject': 'avg-rt'
            }, pass_fail1)

        pass_fail2 = PassFailStatus()

        fc2_triggered = DataCriterion(
            {
                'stop': True,
                'label': 'Sample 2 Triggered',
                'fail': True,
                'timeframe': -1,
                'threshold': '150ms',
                'condition': '<=',
                'subject': 'avg-rt'
            }, pass_fail1)

        fc2_not_triggered = DataCriterion(
            {
                'stop': True,
                'label': 'Sample 2 Not Triggered',
                'fail': True,
                'timeframe': -1,
                'threshold': '300ms',
                'condition': '=',
                'subject': 'avg-rt'
            }, pass_fail1)

        pass_fail1.criteria.append(fc1_triggered)
        pass_fail1.criteria.append(fc1_not_triggered)
        pass_fail2.criteria.append(fc2_triggered)
        pass_fail2.criteria.append(fc2_not_triggered)

        fc1_triggered.is_triggered = True
        fc2_triggered.is_triggered = True

        obj.engine.reporters.append(pass_fail1)
        obj.engine.reporters.append(pass_fail2)
        obj.engine.reporters.append(BlazeMeterUploader())

        path_from_config = tempfile.mktemp(suffix='.xml',
                                           prefix='junit-xml_passfail',
                                           dir=obj.engine.artifacts_dir)

        obj.parameters.merge({
            "filename": path_from_config,
            "data-source": "pass-fail"
        })
        obj.prepare()
        obj.last_second = DataPoint(0)
        obj.post_process()

        with open(obj.report_file_path, 'rb') as fds:
            f_contents = fds.read()

        logging.info("File: %s", f_contents)
        xml_tree = etree.fromstring(f_contents)
        self.assertEqual('testsuites', xml_tree.tag)
        suite = xml_tree.getchildren()[0]
        self.assertEqual('testsuite', suite.tag)
        test_cases = suite.getchildren()
        self.assertEqual(4, len(test_cases))
        self.assertEqual('testcase', test_cases[0].tag)
        self.assertEqual('error', test_cases[0].getchildren()[1].tag)
        self.assertEqual('error', test_cases[2].getchildren()[1].tag)

        sys_out = test_cases[0].getchildren()[0]
        self.assertEqual('system-out', sys_out.tag)
        self.assertIn('BlazeMeter report link: http://test/report/123',
                      sys_out.text)
Ejemplo n.º 27
0
    def test_prepare_label_issue(self):
        # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ
        obj = PassFailStatus()
        obj.parameters = {"criterias": ["avg-rt of spaced label>10ms"]}
        obj.prepare()
        self.assertGreater(len(obj.criterias), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CUMULATIVE]['spaced label'] = point[DataPoint.CUMULATIVE]['']
            point[DataPoint.CURRENT]['spaced label'] = point[DataPoint.CURRENT]['']
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()

        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 28
0
    def test_prepare_label_issue(self):
        # https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/codename-taurus/PWjU7xVucZ0/WkjUAbE1EwAJ
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": ["avg-rt of spaced label>10ms"]}
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            point[DataPoint.CUMULATIVE]['spaced label'] = point[
                DataPoint.CUMULATIVE]['']
            point[DataPoint.CURRENT]['spaced label'] = point[
                DataPoint.CURRENT]['']
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()

        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 29
0
    def test_prepare(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        config = json.loads(open(__dir__() + "/../json/passfail.json").read())
        obj.parameters = config['reporting'][0]
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            logging.info("%s: %s", n, point)
            obj.aggregated_second(point)
            try:
                obj.check()
            except AutomatedShutdown:
                pass

        try:
            obj.post_process()
        except AutomatedShutdown:
            pass
Ejemplo n.º 30
0
    def test_rc_within(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": [
                "rc413>10 within 3s, stop as successful",  # this one to cover branch that caused bug
                "rc413>10 within 10s, stop as failed",
            ]
        }
        obj.prepare()
        self.assertEquals(len(obj.criteria), 2)

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['413'] = 3
            obj.aggregated_second(point)
            try:
                obj.check()
            except AutomatedShutdown:
                break

            self.assertLess(n, 3)

        obj.shutdown()
        obj.post_process()
        self.assertFalse(obj.criteria[0].is_triggered)
        self.assertTrue(obj.criteria[1].is_triggered)
Ejemplo n.º 31
0
    def test_percentiles_track(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {"criteria": ["p90>0ms"]}
        obj.prepare()
        self.assertGreater(len(obj.criteria), 0)

        for n in range(0, 10):
            point = random_datapoint(n)
            obj.aggregated_second(point)
            obj.check()

        obj.shutdown()
        try:
            obj.post_process()
            self.fail()
        except AutomatedShutdown:
            pass
Ejemplo n.º 32
0
    def test_rc_over1(self):
        obj = PassFailStatus()
        obj.engine = EngineEmul()
        obj.parameters = {
            "criteria": [
                "rc200<8 over 5s",
            ]
        }
        obj.prepare()

        for n in range(0, 10):
            point = random_datapoint(n)
            rcs = point[DataPoint.CURRENT][''][KPISet.RESP_CODES]
            rcs['200'] = 3
            obj.aggregated_second(point)
            obj.check()
            self.assertFalse(obj.criteria[0].is_triggered)

        obj.shutdown()
        obj.post_process()
Ejemplo n.º 33
0
    def test_within(self):
        obj = PassFailStatus()
        obj.parameters = {
            "criterias": [
                "fail>10% within 5s",
                "fail>1000 within 5s",
                "avg-rt>100ms within 10s",
            ]
        }
        obj.prepare()

        start_time = time.time()
        for _n in range(0, 20):
            point = random_datapoint(start_time)
            obj.aggregated_second(point)
            if _n % 2 == 0:
                obj.check()
            obj.check()
            start_time += 1