def test_history_monotonous(history_with_epoch_set):
ts = history_with_epoch_set.epoch
for delta in (Timedelta.from_s(1), Timedelta.from_string("20s")):
new_ts = ts + delta
history_with_epoch_set.insert(new_ts, State.OK)
assert history_with_epoch_set.transitions[-1].time == new_ts
def _calculate_interval_durations(self, response_aggregates):
# we assume LAST semantic, but this should not matter for equidistant intervals
yield Timedelta(0)
for previous_ta, current_ta in zip(
response_aggregates, response_aggregates[1:]
):
duration = current_ta.timestamp - previous_ta.timestamp
# We need strong monotony. DB-HTA guarantees it currently
assert duration > Timedelta(0)
yield duration
def __init__(self, time_window: Optional[Timedelta]):
"""A history of state transitions for some metric, spanning at most a
duration of ``time_window``.
It contains a point in time called 'epoch' that signifies when the state
that the first transition switched from was entered. In a graph (t[0]
is the first transition, t[1] the second etc.):
┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄╮┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄╮ ⋯
│ t[0].state │ t[1].state │
⋯──────┴──────────────┴────────────────────┴────────────→
epoch t[0].time t[1].time time
"""
# The list of state transitions for this metric.
self._transitions: List[StateTransition] = list()
# The point in time at which we assume the metric entered the state
# given by the first transition ``self._transitions[0]``, if present.
# This is necessary as transitions have last semantics, and we otherwise
# had to assume that it was in this state since forever (which skews
# statistics significantly, as you can imagine).
self._epoch: Optional[Timestamp] = None
# Use a sensible default time window for keeping past transitions.
self._time_window: Timedelta
if time_window is None:
self._time_window = Timedelta.from_s(30)
else:
if time_window.ns > 0:
self._time_window = time_window
else:
raise ValueError(
"State transition history time window must be a positive duration"
)
async def test_timeout_check_no_bump(timeout_check):
await sleep(timeout_check._timeout + Timedelta.from_ms(10))
callback = timeout_check._timeout_callback
assert callback.called
assert callback.last_timestamp is None
assert callback.timeout == timeout_check._timeout
def timedelta_random_list():
"""Generate a stream of random Timedeltas of different magnitudes, with
varying amounts of trailing zeroes. The durations (in nanoseconds) look
like this::
1 to 17 digits total
(num_digits)
|
.----------+----------.
xxxxxxxxx00000000000000
'---+---'
|
1 to num_digits
leading random digits
(num_leading_digits)
"""
rng = Random(72438990202596743)
for num_digits in range(1, 17):
for num_leading_digits in range(1, num_digits):
random = rng.randrange(10**num_leading_digits,
10**(num_leading_digits + 1))
ts = random * (10**(num_digits - num_leading_digits))
logger.debug(f"num_digits={num_digits:2}, "
f"num_leading_digits={num_leading_digits:2}, "
f"random={random:20}, "
f"ts={ts}, ")
yield Timedelta(ts)
def test_history_insert_with_epoch_set(history_with_epoch_set):
# Inserting a transition into a history with an epoch already set should
# make that transition the most recent transition of that history.
ts = history_with_epoch_set.epoch + Timedelta.from_s(1)
history_with_epoch_set.insert(ts, State.OK)
assert history_with_epoch_set.transitions[-1].time == ts
def state_prevalences(self) -> Optional[Dict[State, float]]:
"""Return a ``dict`` where for each state, the share of time that a
metric was in this state is a ``float`` between ``0.0`` and ``1.0``.
"""
# We might only calculate prevalences of states if we already set an
# epoch and there exists at least one transition.
if self.is_empty():
return None
# Determine the first timestamp for calculating the cumulative duration
# of each state. Make sure it is at most self._time_window in the past,
# wrt. the most recent transition in this history.
latest_transition = self._transitions[-1]
oldest_transition_time = max(
latest_transition.time - self._time_window, self._epoch)
total_duration: Timedelta = latest_transition.time - oldest_transition_time
cumulative_durations = {state: Timedelta(0) for state in State}
prev_time: Timestamp = oldest_transition_time
current: StateTransition
for current in self._transitions:
cumulative_durations[current.state] += current.time - prev_time
prev_time = current.time
try:
# Return the prevalence of each state as a percentage of the total
# time spanned by all transitions.
return {
state: duration.ns / total_duration.ns
for state, duration in cumulative_durations.items()
}
except ZeroDivisionError:
return None
def test_history_non_monotonous(history_with_epoch_set, delta, caplog):
ts = history_with_epoch_set.epoch + Timedelta.from_s(1)
history_with_epoch_set.insert(ts, State.OK)
next_ts = ts + delta
with caplog.at_level(logging.WARNING):
history_with_epoch_set.insert(next_ts, State.OK)
assert "Times of state transitions must be strictly increasing" in caplog.text
async def test_timeout_check_no_callback_after_cancel(timeout):
class CallOnce(Callback):
def __init__(self):
self.called_before = False
super().__init__()
def __call__(self, *, timeout, last_timestamp):
assert not self.called_before
super().__call__(timeout=timeout, last_timestamp=last_timestamp)
self.called_before = True
timeout_check = TimeoutCheck(timeout=timeout, on_timeout=CallOnce())
timeout_check.start()
await sleep(timeout_check._timeout + Timedelta.from_ms(10))
timeout_check.cancel()
await sleep(timeout_check._timeout + Timedelta.from_ms(10))
def _metric():
timestamp = Timestamp(0)
delta = Timedelta.from_s(1)
value = 0.0
while True:
yield (timestamp, value)
timestamp += delta
value += 1.0
def __init__(
self,
timeout: Timedelta,
on_timeout: Coroutine,
grace_period: Timedelta = Timedelta(0),
):
self._timeout = timeout
self._on_timeout_callback = on_timeout
self._grace_period = grace_period
self._last_timestamp: Optional[Timestamp] = None
self._new_timestamp_event: Event = Event()
self._task: asyncio.Task = asyncio.create_task(self._run())
self._throttle = False
class Ticker:
DEFAULT_DELTA = Timedelta.from_s(1)
DEFAULT_START = Timestamp(0)
def __init__(self, delta=DEFAULT_DELTA, start=DEFAULT_START):
self.delta = delta
self.start = start
self.now = self.start
def __next__(self):
now = self.now
self.now += self.delta
return now
def __iter__(self):
return self
def __init__(
self,
timeout: Timedelta,
on_timeout: TimeoutCallback,
grace_period: Optional[Timedelta] = None,
name: Optional[str] = None,
):
self._timeout = timeout
self._timeout_callback: TimeoutCallback = on_timeout
self._grace_period = Timedelta(
0) if grace_period is None else grace_period
self._name = name
self._last_timestamp: Optional[Timestamp] = None
self._new_timestamp_event: Event = Event()
self._throttle = False
def __init__(
self,
name: str,
metrics: Iterable[str],
value_constraints: Optional[Dict[str, float]],
timeout: Optional[str] = None,
on_timeout: Optional[Coroutine] = None,
):
"""Create value- and timeout-checks for a set of metrics
:param name: The name of this check
:param metrics: Iterable of names of metrics to monitor
:param value_constraints: Dictionary indicating warning and critical
value ranges, see ValueCheck. If omitted, this check does not care
for which values its metrics report.
:param timeout: If set, and a metric does not deliver values within
this duration, run the callback on_timeout
:param on_timeout: Callback to run when metrics do not deliver values
in time, mandatory if timeout is given.
"""
self._name = name
self._metrics: Set[str] = set(metrics)
self._status_cache = StatusCache(self._metrics)
self._value_checks: Optional[Dict[str, ValueCheck]] = None
self._timeout_checks: Optional[Dict[str, TimeoutCheck]] = None
self._on_timeout_callback: Optional[Coroutine] = None
if value_constraints is not None:
self._value_checks: Dict[str, ValueCheck] = {
metric: ValueCheck(**value_constraints) for metric in self._metrics
}
if timeout is not None:
if on_timeout is None:
raise ValueError("on_timeout callback is required if timeout is given")
self._on_timeout_callback = on_timeout
self._timeout_checks = {
metric: TimeoutCheck(
Timedelta.from_string(timeout),
self._get_on_timeout_callback(metric),
)
for metric in self._metrics
}
def test_timeaggregate_from_value(timestamp):
VALUE = 42.0
agg = TimeAggregate.from_value(timestamp=timestamp, value=VALUE)
assert agg.timestamp == timestamp
assert agg.active_time == Timedelta(0)
assert agg.count == 1
assert agg.minimum == VALUE
assert agg.maximum == VALUE
assert agg.sum == VALUE
assert agg.integral_ns == 0
assert isclose(agg.mean, VALUE)
assert isclose(agg.mean_sum, VALUE)
with pytest.raises(ZeroDivisionError):
agg.mean_integral
def convert(
self,
value: Union[str, Timedelta],
param: Optional[Parameter],
ctx: Optional[Context],
) -> Optional[Timedelta]:
if value is None:
return None
elif isinstance(value, str):
try:
return Timedelta.from_string(value)
except ValueError:
self.fail(
'expected a duration: "<value>[<unit>]"',
param=param,
ctx=ctx,
)
else:
return value
def parse_functions(target_dict):
for function in target_dict.get("functions", ["avg"]):
if function == "avg":
yield AvgFunction()
elif function == "min":
yield MinFunction()
elif function == "max":
yield MaxFunction()
elif function == "count":
yield CountFunction()
elif function == "sma":
try:
yield MovingAverageFunction(
Timedelta.from_string(target_dict.get("sma_window"))
)
except (TypeError, KeyError):
pass
# Cannot instantiate RawFunction - it automatically replaces the aggregates when zooming in
else:
raise KeyError(f"Unknown function '{function}' requested")
def soft_fail_cache():
return StateCache(
metrics=["publish.rate"],
transition_debounce_window=Timedelta.from_string("1 day"),
transition_postprocessor=SoftFail(max_fail_count=2),
)
def __init__(self, minimum_duration: Union[Timedelta, str], **_kwargs):
self._minimum_duration = (Timedelta.from_string(minimum_duration)
if isinstance(minimum_duration, str) else
minimum_duration)
def time_delta_random():
return Timedelta(8295638928)
def test_timedelta_floordiv(ns: int, factor: int, expected_ns: int):
timedelta = Timedelta(ns)
assert (timedelta // factor) == Timedelta(expected_ns)
assert history_with_epoch_set.transitions[-1].time == ts
def test_history_monotonous(history_with_epoch_set):
ts = history_with_epoch_set.epoch
for delta in (Timedelta.from_s(1), Timedelta.from_string("20s")):
new_ts = ts + delta
history_with_epoch_set.insert(new_ts, State.OK)
assert history_with_epoch_set.transitions[-1].time == new_ts
@pytest.mark.parametrize(
"delta",
[
Timedelta(0),
Timedelta.from_s(-1),
],
)
def test_history_non_monotonous(history_with_epoch_set, delta, caplog):
ts = history_with_epoch_set.epoch + Timedelta.from_s(1)
history_with_epoch_set.insert(ts, State.OK)
next_ts = ts + delta
with caplog.at_level(logging.WARNING):
history_with_epoch_set.insert(next_ts, State.OK)
assert "Times of state transitions must be strictly increasing" in caplog.text
@pytest.mark.parametrize(
"ticker",
def timeout() -> Timedelta:
return Timedelta.from_ms(100)
def test_timedelta_truediv(ns: int, factor: Union[int, float],
expected_ns: int):
timedelta = Timedelta(ns)
assert (timedelta / factor) == Timedelta(expected_ns)
rpc: AsyncMock
@pytest.fixture
def interval_source():
with patch("metricq.interval_source.IntervalSource.rpc"):
source = _TestIntervalSource(token="source-interval-test",
management_url="amqps://test.invalid")
yield source
@pytest.mark.parametrize(
("period", "normalized"),
[
(Timedelta(1337), Timedelta(1337)),
(4, Timedelta.from_s(4)),
],
)
def test_period_setter_normalizing(interval_source: _TestIntervalSource,
period, normalized):
"""Internally, the interval source period is normalized to a Timedelta"""
assert interval_source.period is None
interval_source.period = period
assert interval_source.period == normalized
def test_period_no_reset(interval_source: _TestIntervalSource):
"""Currently, the interval source period cannot be reset to None"""
def tick() -> Timedelta:
return Timedelta.from_s(0.1)
def __init__(self):
self.interval = Timedelta(0)
def transform_data(self, response):
if len(response) == 0:
return []
response_aggregates = list(response.aggregates(convert=True))
ma_integral_ns = 0
ma_active_time = Timedelta(0)
ma_begin_index = 1
ma_begin_time = response_aggregates[0].timestamp
ma_end_index = 1
ma_end_time = response_aggregates[0].timestamp
interval_durations = list(
self._calculate_interval_durations(response_aggregates)
)
for timeaggregate, current_interval_duration in zip(
response_aggregates, interval_durations
):
# The moving average window is symmetric around the current *interval* - not the current point
# How much time is covered by the current interval width and how much is on both sides "outside"
assert current_interval_duration >= Timedelta(0)
outside_duration = self.interval - current_interval_duration
# If the current interval is wider than the target moving average window, just use the current one
outside_duration = max(Timedelta(0), outside_duration)
seek_begin_time = (
timeaggregate.timestamp
- current_interval_duration
- outside_duration / 2
)
seek_end_time = timeaggregate.timestamp + outside_duration / 2
# TODO unify these two loops
# Move left part of the window
while ma_begin_time < seek_begin_time:
next_step_time = min(
response_aggregates[ma_begin_index].timestamp, seek_begin_time
)
step_duration = next_step_time - ma_begin_time
# scale can be 0 (everything is nop),
# 1 (full interval needs to be removed), or something in between
scale = step_duration.ns / interval_durations[ma_begin_index].ns
ma_active_time -= (
response_aggregates[ma_begin_index].active_time * scale
)
ma_integral_ns -= (
response_aggregates[ma_begin_index].integral_ns * scale
)
ma_begin_time = next_step_time
assert ma_begin_time <= response_aggregates[ma_begin_index].timestamp
if ma_begin_time == response_aggregates[ma_begin_index].timestamp:
# Need to move to the next interval
ma_begin_index += 1
# Move right part of the window
while ma_end_time < seek_end_time and ma_end_index < len(
response_aggregates
):
next_step_time = min(
response_aggregates[ma_end_index].timestamp, seek_end_time
)
step_duration = next_step_time - ma_end_time
# scale can be 0 (everything is nop),
# 1 (full interval needs to be removed), or something in between
scale = step_duration.ns / interval_durations[ma_end_index].ns
ma_active_time += response_aggregates[ma_end_index].active_time * scale
ma_integral_ns += response_aggregates[ma_end_index].integral_ns * scale
ma_end_time = next_step_time
assert ma_end_time <= response_aggregates[ma_end_index].timestamp
if ma_end_time == response_aggregates[ma_end_index].timestamp:
# Need to move to the next interval
ma_end_index += 1
if seek_begin_time != ma_begin_time or seek_end_time != ma_end_time:
# Interval window not complete
continue
if ma_active_time == 0:
continue
yield timeaggregate.timestamp, ma_integral_ns / ma_active_time.ns
def test_duration_param():
value = "30s"
DURATION = DurationParam(default=None)
assert DURATION.convert(value, param=None,
ctx=None) == Timedelta.from_string(value)
def test_timedelta_from_string(input, expected_ns):
assert Timedelta.from_string(input) == Timedelta(expected_ns)