def test_validate_exclusion(self):
threshold = 100.0
composite_metric = CompositeMetric(name="TestMetric")
metric = Metric(name="fail1", value=99.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=110.0)
composite_metric.add(metric)
# Same test as above, but with exclusions:
rule = Rule("/TestMetric#fail1",
operation=Rule.Evaluation.GREATER_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric, exclusions={"/TestMetric*fail1"})
def test_add_validation(self):
rule = Rule(pattern="/TestMetrics/child1#metric1",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=0.12)
rules_set = RulesEngine.RuleSet()
rules_set.add_validation(rule)
assert rule not in rules_set._alerts
assert rule in rules_set._validations
def mock_validate(self,
composite_metric,
previous_metric=None,
exclusions=None):
nonlocal validations
assert exclusions == {"/CodeCoverage/by_file*test_excluded.py"
} or len(exclusions) == 0
if fnmatch.fnmatchcase(self._pattern,
"/CodeCoverage/by_file*test_excluded.py"):
return
validations.append((self, composite_metric))
if self._pattern == "/CodeCoverage#overall" and self._limit > 81.0:
raise Rule.ThresholdViolation(
msg="failure_validation_codecov",
parent=composite_metric,
offending_elements=['/CodeCoverage#overall'])
elif self._pattern == "/Performance#overall_cpu":
raise Rule.ThresholdViolation(
msg="second_alert_performance",
parent=composite_metric,
offending_elements=['/Performance#overall_cpu'])
def test_validate_gt(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=90.0)
composite_metric.add(metric)
metric = Metric(name="fail2", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass", value=110.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in str(e)
assert "#fail2" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail2 <= {threshold}" in str(e)
assert "#fail1" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in f"{e}"
assert f"/Test_Metric#fail2 <= {threshold}" in f"{e}"
assert "#pass" not in f"{e}"
def test_validate_lte(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=110.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=99.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in str(e)
assert "#pass1" not in str(e)
assert "#pass2" not in str(e)
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
# should not raise exception:
rule.validate(composite_metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in f"{e}"
assert "#pass1" not in f"{e}"
assert "#pass2" not in f"{e}"