def test_validate_exclusion(self): threshold = 100.0 composite_metric = CompositeMetric(name="TestMetric") metric = Metric(name="fail1", value=99.0) composite_metric.add(metric) metric = Metric(name="pass1", value=threshold) composite_metric.add(metric) metric = Metric(name="pass2", value=110.0) composite_metric.add(metric) # Same test as above, but with exclusions: rule = Rule("/TestMetric#fail1", operation=Rule.Evaluation.GREATER_THAN_OR_EQUAL, limiting_value=threshold) rule.validate(composite_metric, exclusions={"/TestMetric*fail1"})
def test_add_validation(self): rule = Rule(pattern="/TestMetrics/child1#metric1", operation=Rule.Evaluation.GREATER_THAN, limiting_value=0.12) rules_set = RulesEngine.RuleSet() rules_set.add_validation(rule) assert rule not in rules_set._alerts assert rule in rules_set._validations
def mock_validate(self, composite_metric, previous_metric=None, exclusions=None): nonlocal validations assert exclusions == {"/CodeCoverage/by_file*test_excluded.py" } or len(exclusions) == 0 if fnmatch.fnmatchcase(self._pattern, "/CodeCoverage/by_file*test_excluded.py"): return validations.append((self, composite_metric)) if self._pattern == "/CodeCoverage#overall" and self._limit > 81.0: raise Rule.ThresholdViolation( msg="failure_validation_codecov", parent=composite_metric, offending_elements=['/CodeCoverage#overall']) elif self._pattern == "/Performance#overall_cpu": raise Rule.ThresholdViolation( msg="second_alert_performance", parent=composite_metric, offending_elements=['/Performance#overall_cpu'])
def test_validate_gt(self): threshold = 100.0 composite_metric = CompositeMetric(name="Test_Metric") metric = Metric(name="fail1", value=90.0) composite_metric.add(metric) metric = Metric(name="fail2", value=threshold) composite_metric.add(metric) metric = Metric(name="pass", value=110.0) composite_metric.add(metric) with pytest.raises(Rule.ThresholdViolation) as e: rule = Rule("/Test_Metric#fail1", operation=Rule.Evaluation.GREATER_THAN, limiting_value=threshold) rule.validate(composite_metric) assert f"/Test_Metric#fail1 <= {threshold}" in str(e) assert "#fail2" not in str(e) assert "#pass" not in str(e) with pytest.raises(Rule.ThresholdViolation) as e: rule = Rule("/Test_Metric#fail2", operation=Rule.Evaluation.GREATER_THAN, limiting_value=threshold) rule.validate(composite_metric) assert f"/Test_Metric#fail2 <= {threshold}" in str(e) assert "#fail1" not in str(e) assert "#pass" not in str(e) with pytest.raises(Rule.ThresholdViolation) as e: rule = Rule("/Test_Metric#*", operation=Rule.Evaluation.GREATER_THAN, limiting_value=threshold) rule.validate(composite_metric) assert f"/Test_Metric#fail1 <= {threshold}" in f"{e}" assert f"/Test_Metric#fail2 <= {threshold}" in f"{e}" assert "#pass" not in f"{e}"
def test_validate_lte(self): threshold = 100.0 composite_metric = CompositeMetric(name="Test_Metric") metric = Metric(name="fail1", value=110.0) composite_metric.add(metric) metric = Metric(name="pass1", value=threshold) composite_metric.add(metric) metric = Metric(name="pass2", value=99.0) composite_metric.add(metric) with pytest.raises(Rule.ThresholdViolation) as e: rule = Rule("/Test_Metric#fail1", operation=Rule.Evaluation.LESS_THAN_OR_EQUAL, limiting_value=threshold) rule.validate(composite_metric) assert f"/Test_Metric#fail1 > {threshold}" in str(e) assert "#pass1" not in str(e) assert "#pass2" not in str(e) rule = Rule("/Test_Metric#fail2", operation=Rule.Evaluation.LESS_THAN_OR_EQUAL, limiting_value=threshold) # should not raise exception: rule.validate(composite_metric) with pytest.raises(Rule.ThresholdViolation) as e: rule = Rule("/Test_Metric#*", operation=Rule.Evaluation.LESS_THAN_OR_EQUAL, limiting_value=threshold) rule.validate(composite_metric) assert f"/Test_Metric#fail1 > {threshold}" in f"{e}" assert "#pass1" not in f"{e}" assert "#pass2" not in f"{e}"