def test_to_dataclass_simple(self):
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
metric.add(Metric("two", 1.0))
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: float
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
def test_keys(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
assert set(root.keys(core_metrics_only=True)) == {
"child_test_metric/grandchild_test_metric#test_metric3",
"#pathed/test_metric2"}
assert set(root.keys()) == {
"child_test_metric/grandchild_test_metric#test_metric3",
"#pathed/test_metric2",
"child_test_metric",
"child_test_metric/grandchild_test_metric"}
for key in root.keys(core_metrics_only=False):
if '#' in key:
assert isinstance(root[key], Metric)
else:
assert isinstance(root[key], CompositeMetric)
assert root["child_test_metric"] == child
with pytest.raises(KeyError):
root["no_such_child"]
def test_from_dict(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
result = root.flatten()
new_root = BasicMetric.from_flattened(result)
assert result == new_root.flatten()
def test_validate_gt(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=90.0)
composite_metric.add(metric)
metric = Metric(name="fail2", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass", value=110.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in str(e)
assert "#fail2" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail2 <= {threshold}" in str(e)
assert "#fail1" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in f"{e}"
assert f"/Test_Metric#fail2 <= {threshold}" in f"{e}"
assert "#pass" not in f"{e}"
def test_validate_lte(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=110.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=99.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in str(e)
assert "#pass1" not in str(e)
assert "#pass2" not in str(e)
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
# should not raise exception:
rule.validate(composite_metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in f"{e}"
assert "#pass1" not in f"{e}"
assert "#pass2" not in f"{e}"
def add_by_test(self, test_name: str, user_cpu_secs: float,
system_cpu_secs: float, duration: float):
composite = CompositeMetric(name=test_name)
composite.add(Metric(name="user_cpu", value=user_cpu_secs))
composite.add(Metric(name="sys_cpu", value=system_cpu_secs))
composite.add(Metric(name="duration", value=duration))
self.by_test.add(composite)
def test_flatten(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
result = root.flatten()
assert len(root.value) == 2
assert '/root_test_metric/child_test_metric/grandchild_test_metric#test_metric3' in result
assert result['/root_test_metric/child_test_metric/grandchild_test_metric#test_metric3'] == test_value3
assert '/root_test_metric#pathed/test_metric2' in result
assert result['/root_test_metric#pathed/test_metric2'] == test_value2
def data_generator():
seed = [1, 28832.12993, 0.00081238, 291]
for index in range(100):
top = CompositeMetric(name="TestMetric")
child1 = Metric("child1", seed[0])
child2 = CompositeMetric("child2")
child3 = CompositeMetric("child3")
top.add(child1)
top.add(child2)
top.add(child3)
grandchild2_1 = Metric("grandchild1", seed[1])
grandchild2_2 = Metric("grandchild2", seed[2])
child2.add(grandchild2_1)
child2.add(grandchild2_2)
grandchild3_1 = Metric("grandchild1", seed[3])
child3.add(grandchild3_1)
yield top
seed[0] += 1
seed[1] *= 0.9992
seed[2] *= 1.2
seed[3] -= 2
def test_validate_exclusion(self):
threshold = 100.0
composite_metric = CompositeMetric(name="TestMetric")
metric = Metric(name="fail1", value=99.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=110.0)
composite_metric.add(metric)
# Same test as above, but with exclusions:
rule = Rule("/TestMetric#fail1",
operation=Rule.Evaluation.GREATER_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric, exclusions={"/TestMetric*fail1"})
def test_to_dataclass_simple_with_dict(self):
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
metric.add(Metric("two", 1.0))
inner = CompositeMetric("values")
comp_inner1 = CompositeMetric("comp_one")
comp_inner1.add(Metric("in_one", 23.354))
comp_inner1.add(Metric("in_two", -23.354))
inner.add(comp_inner1)
comp_inner2 = CompositeMetric("comp_two")
comp_inner2.add(Metric("in_one", 3455623.354))
comp_inner2.add(Metric("in_two", -56576823.354))
inner.add(comp_inner2)
metric.add(inner)
@dataclass
class Inner(MetricDataClass):
in_one: float
in_two: float
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: float
values: Dict[str, Inner]
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
assert data.values["comp_two"].in_one == inner.value["comp_two"].value[
"in_one"].value
assert data.values["comp_two"].in_two == inner.value["comp_two"].value[
"in_two"].value
def test_to_dataclass_simple_with_dict_and_optionals(self):
@dataclass
class Inner(MetricDataClass):
in_one: float
in_two: Optional[float] = None
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: Optional[float] = None
values: Optional[Dict[str, Inner]] = None
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two is None
assert data.values is None
metric.add(Metric("two", 1.0))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert data.values is None
inner = CompositeMetric("values")
comp_inner1 = CompositeMetric("comp_one")
comp_inner1.add(Metric("in_one", 23.354))
comp_inner1.add(Metric("in_two", -23.354))
inner.add(comp_inner1)
metric.add(inner)
metric.add(Metric("two", 1.0))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
comp_inner2 = CompositeMetric("comp_two")
comp_inner2.add(Metric("in_one", 3455623.354))
inner.add(comp_inner2)
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
assert data.values["comp_two"].in_one == inner.value["comp_two"].value[
"in_one"].value
assert data.values["comp_two"].in_two == None