def test_validate_gt(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=90.0)
composite_metric.add(metric)
metric = Metric(name="fail2", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass", value=110.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in str(e)
assert "#fail2" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail2 <= {threshold}" in str(e)
assert "#fail1" not in str(e)
assert "#pass" not in str(e)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.GREATER_THAN,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 <= {threshold}" in f"{e}"
assert f"/Test_Metric#fail2 <= {threshold}" in f"{e}"
assert "#pass" not in f"{e}"
def add_by_test(self, test_name: str, user_cpu_secs: float,
system_cpu_secs: float, duration: float):
composite = CompositeMetric(name=test_name)
composite.add(Metric(name="user_cpu", value=user_cpu_secs))
composite.add(Metric(name="sys_cpu", value=system_cpu_secs))
composite.add(Metric(name="duration", value=duration))
self.by_test.add(composite)
def test_validate_lte(self):
threshold = 100.0
composite_metric = CompositeMetric(name="Test_Metric")
metric = Metric(name="fail1", value=110.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=99.0)
composite_metric.add(metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#fail1",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in str(e)
assert "#pass1" not in str(e)
assert "#pass2" not in str(e)
rule = Rule("/Test_Metric#fail2",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
# should not raise exception:
rule.validate(composite_metric)
with pytest.raises(Rule.ThresholdViolation) as e:
rule = Rule("/Test_Metric#*",
operation=Rule.Evaluation.LESS_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric)
assert f"/Test_Metric#fail1 > {threshold}" in f"{e}"
assert "#pass1" not in f"{e}"
assert "#pass2" not in f"{e}"
def test_validate_exclusion(self):
threshold = 100.0
composite_metric = CompositeMetric(name="TestMetric")
metric = Metric(name="fail1", value=99.0)
composite_metric.add(metric)
metric = Metric(name="pass1", value=threshold)
composite_metric.add(metric)
metric = Metric(name="pass2", value=110.0)
composite_metric.add(metric)
# Same test as above, but with exclusions:
rule = Rule("/TestMetric#fail1",
operation=Rule.Evaluation.GREATER_THAN_OR_EQUAL,
limiting_value=threshold)
rule.validate(composite_metric, exclusions={"/TestMetric*fail1"})
def validate(self,
composite_metric: CompositeMetric,
previous_metric: Optional[CompositeMetric] = None,
exclusions: Optional[Iterable[str]] = None) -> None:
"""
Valide a composite metric for any and all matching key-names for each of its components
:param composite_metric: the `CompositMetric` to validate
:param previous_metric: previous value of the metric (for relative rule) or None if N/A or non-existent
:raises: ValueError with a message containing the rules violated if the metric fails to validate
against this rule
"""
failed_elements: List[str] = []
msg = ""
for key in [
k for k in composite_metric.keys(core_metrics_only=True)
if not self._excluded(self._prepend_root(k, composite_metric),
exclusions) and
(self._pattern == '*' or fnmatch.fnmatchcase(
self._prepend_root(k, composite_metric), self._pattern))
]:
if self._is_relative:
if not previous_metric:
continue
try:
value = composite_metric[key].value - previous_metric[
key].value
except KeyError:
continue # prev metric does not contain this key, so nothing to compare to
else:
value = composite_metric[key].value
if self._operation == Rule.Evaluation.LESS_THAN:
if value >= self._limit:
msg += f"\n {self._prepend_root(key, composite_metric)} >= {self._limit}"
failed_elements.append(key)
elif self._operation == Rule.Evaluation.GREATER_THAN:
if value <= self._limit:
msg += f"\n {self._prepend_root(key, composite_metric)} <= {self._limit}"
failed_elements.append(key)
elif self._operation == Rule.Evaluation.LESS_THAN_OR_EQUAL:
if value > self._limit:
failed_elements.append(key)
msg += f"\n {self._prepend_root(key, composite_metric)} > {self._limit}"
elif self._operation == Rule.Evaluation.GREATER_THAN_OR_EQUAL:
if value < self._limit:
failed_elements.append(key)
msg += f"\n {self._prepend_root(key, composite_metric)} < {self._limit}"
if failed_elements:
raise Rule.ThresholdViolation(msg=msg,
parent=composite_metric,
offending_elements=failed_elements)
def test_to_dataclass_simple(self):
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
metric.add(Metric("two", 1.0))
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: float
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
def execute(self) -> QueryResult[MetricDataClassT]:
# we order timestamps for query in descending order to filter out "the top" which are the newest items
self._statement = self._statement.order_by(
desc(SQLCompositeMetric.timestamp))
if self._max_count:
self._statement = self._statement.limit(self._max_count)
sql_result: List[SQLCompositeMetric] = self._statement.all()
result: QueryResult[MetricDataClassT] = QueryResult()
for item in reversed(
sql_result
): # order timestamps from oldest to newest when returning to client
flattened: Dict[str, float] = {}
for child in item.children:
flattened[child.name] = child.value
metadata = Metadata({
data.name: data.value
for data in item.metrics_metadata.data
})
result.metadata.append(metadata)
result.timestamps.append(item.timestamp)
result.metric_data.append(
CompositeMetric.from_flattened(flattened).to_dataclass(
self._type))
return result
def test_to_dataclass_simple_with_dict(self):
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
metric.add(Metric("two", 1.0))
inner = CompositeMetric("values")
comp_inner1 = CompositeMetric("comp_one")
comp_inner1.add(Metric("in_one", 23.354))
comp_inner1.add(Metric("in_two", -23.354))
inner.add(comp_inner1)
comp_inner2 = CompositeMetric("comp_two")
comp_inner2.add(Metric("in_one", 3455623.354))
comp_inner2.add(Metric("in_two", -56576823.354))
inner.add(comp_inner2)
metric.add(inner)
@dataclass
class Inner(MetricDataClass):
in_one: float
in_two: float
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: float
values: Dict[str, Inner]
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
assert data.values["comp_two"].in_one == inner.value["comp_two"].value[
"in_one"].value
assert data.values["comp_two"].in_two == inner.value["comp_two"].value[
"in_two"].value
def test_to_dataclass_simple_with_dict_and_optionals(self):
@dataclass
class Inner(MetricDataClass):
in_one: float
in_two: Optional[float] = None
@dataclass
class TestDataClass(MetricDataClass):
one: float
two: Optional[float] = None
values: Optional[Dict[str, Inner]] = None
metric = CompositeMetric("TestMetric")
metric.add(Metric("one", 93.224556768))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two is None
assert data.values is None
metric.add(Metric("two", 1.0))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert data.values is None
inner = CompositeMetric("values")
comp_inner1 = CompositeMetric("comp_one")
comp_inner1.add(Metric("in_one", 23.354))
comp_inner1.add(Metric("in_two", -23.354))
inner.add(comp_inner1)
metric.add(inner)
metric.add(Metric("two", 1.0))
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
comp_inner2 = CompositeMetric("comp_two")
comp_inner2.add(Metric("in_one", 3455623.354))
inner.add(comp_inner2)
data = metric.to_dataclass(TestDataClass)
assert data.one == metric["one"].value
assert data.two == metric["two"].value
assert isinstance(data.values["comp_one"], Inner)
assert data.values["comp_one"].in_one == inner.value["comp_one"].value[
"in_one"].value
assert data.values["comp_one"].in_two == inner.value["comp_one"].value[
"in_two"].value
assert data.values["comp_two"].in_one == inner.value["comp_two"].value[
"in_one"].value
assert data.values["comp_two"].in_two == None
def test_invalid_name_raises_ValueError(self):
with pytest.raises(ValueError):
CompositeMetric("name_with_#")
with pytest.raises(ValueError):
CompositeMetric("name_with_/")
def test_keys(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
assert set(root.keys(core_metrics_only=True)) == {
"child_test_metric/grandchild_test_metric#test_metric3",
"#pathed/test_metric2"}
assert set(root.keys()) == {
"child_test_metric/grandchild_test_metric#test_metric3",
"#pathed/test_metric2",
"child_test_metric",
"child_test_metric/grandchild_test_metric"}
for key in root.keys(core_metrics_only=False):
if '#' in key:
assert isinstance(root[key], Metric)
else:
assert isinstance(root[key], CompositeMetric)
assert root["child_test_metric"] == child
with pytest.raises(KeyError):
root["no_such_child"]
def test_from_dict(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
result = root.flatten()
new_root = BasicMetric.from_flattened(result)
assert result == new_root.flatten()
def test_flatten(self):
root = CompositeMetric("root_test_metric")
child = CompositeMetric("child_test_metric")
test_value2 = 3490.223
test_value3 = -123.872
root.add(child)
root.add(Metric("pathed/test_metric2", test_value2))
grandchild = CompositeMetric("grandchild_test_metric")
child.add(grandchild)
grandchild.add(Metric("test_metric3", test_value3))
result = root.flatten()
assert len(root.value) == 2
assert '/root_test_metric/child_test_metric/grandchild_test_metric#test_metric3' in result
assert result['/root_test_metric/child_test_metric/grandchild_test_metric#test_metric3'] == test_value3
assert '/root_test_metric#pathed/test_metric2' in result
assert result['/root_test_metric#pathed/test_metric2'] == test_value2
def by_test(self):
if self._by_test is None:
self._by_test = CompositeMetric(name="by_test")
return self._by_test
def by_pkg_composite(self):
if self._by_pkg_metrics is None:
self._by_pkg_metrics = CompositeMetric(name="by_file")
return self._by_pkg_metrics
def data_generator():
seed = [1, 28832.12993, 0.00081238, 291]
for index in range(100):
top = CompositeMetric(name="TestMetric")
child1 = Metric("child1", seed[0])
child2 = CompositeMetric("child2")
child3 = CompositeMetric("child3")
top.add(child1)
top.add(child2)
top.add(child3)
grandchild2_1 = Metric("grandchild1", seed[1])
grandchild2_2 = Metric("grandchild2", seed[2])
child2.add(grandchild2_1)
child2.add(grandchild2_2)
grandchild3_1 = Metric("grandchild1", seed[3])
child3.add(grandchild3_1)
yield top
seed[0] += 1
seed[1] *= 0.9992
seed[2] *= 1.2
seed[3] -= 2