async def get_counter_data(app, metric, start, stop, width):
time_begin = timer()
target = Target(metric, order_time_value=True)
start_time = Timestamp(start * 10**6)
end_time = Timestamp(stop * 10**6)
interval = (end_time - start_time) / width
results, metadata = await asyncio.gather(
target.get_response(app, start_time, end_time, interval),
target.get_metadata(app),
)
result = results[0] if len(results) > 0 else {"datapoints": []}
datapoints = [
datapoint for datapoint in result["datapoints"]
if start <= datapoint[0] <= stop
]
rv = {
"description": metadata.get("description", ""),
"unit": metadata.get("unit", ""),
"data": datapoints,
}
time_diff = timer() - time_begin
logger.log(
logging.DEBUG if time_diff < 1 else logging.INFO,
"get_counter_data for {} returned {} values and took {} s",
metric,
len(rv["data"]),
time_diff,
)
return rv
def test_history_set_epoch(empty_history):
# An empty StateTransitionHistory should not have an epoch set
assert empty_history.epoch is None
# The first transition to be inserted is discarded, only its timestamp is
# kept as the epoch value for this history.
empty_history.insert(Timestamp(0), State.OK)
assert empty_history.epoch == Timestamp(0)
assert not empty_history.transitions
async def _run(self):
logger.info("{!r}: started", self)
try:
while True:
if self._last_timestamp is None or self._throttle:
# We either never got bumped or we just missed a deadline
# and ran the timeout callback. Wait for the entire
# timeout duration in either case, so that we don't spam
# the timeout callback.
timeout = self._timeout + self._grace_period
await self._run_timeout_callback_after(timeout)
else:
# Calculate a deadline by which we expect the next bump,
# based on the last time at which we got bumped.
# We assume our local clock and the clock source for the
# last timestamp to be synchronized within the grace period.
# If the deadline is in the past, immediately run the
# timeout callback.
now = Timestamp.now()
deadline = self._last_timestamp + self._timeout + self._grace_period
if deadline <= now:
logger.debug("{!r}: deadline in the past!", self)
self._run_timeout_callback()
self._throttle = True
else:
wait_duration = deadline - now
await self._run_timeout_callback_after(wait_duration)
except CancelledError:
logger.info("{!r}: stopped", self)
raise
except Exception as e: # pylint: disable=broad-except
logger.exception(
"{!r}: unexpected error inside TimeoutCheck callback: {}",
self, e)
async def test_timeout_check_bump_once(timeout_check: TimeoutCheck):
now = Timestamp.now()
timeout_check.bump(now)
await step()
callback = cast(Callback, timeout_check._timeout_callback)
assert not callback.called
assert timeout_check._last_timestamp == now
def _metric():
timestamp = Timestamp(0)
delta = Timedelta.from_s(1)
value = 0.0
while True:
yield (timestamp, value)
timestamp += delta
value += 1.0
async def test_source_send_error_value_is_none(source_metric: SourceMetric):
async def send(metric, chunk: DataChunk):
assert len(chunk.value) == 1 and isnan(chunk.value[0])
# Replace the send call by a mock and inspect it there. We cannot inspect
# the chunk from mock_calls after the call to error since the chunk will
# have been reset already, so it would always be empty.
source_metric.source.attach_mock(AsyncMock(side_effect=send), "_send")
await source_metric.error(Timestamp(0))
async def get_history_data(app, request):
time_begin = timer()
targets = []
for target_dict in request["targets"]:
metrics = [target_dict["metric"]]
# guess if this is a pattern (regex) to expand by sending it to the manager
if "(" in target_dict["metric"] and ")" in target_dict["metric"]:
metrics = await app["history_client"].get_metrics(
metadata=False, historic=True, selector=target_dict["metric"])
for metric in metrics:
targets.append(
Target(
metric=metric,
name=target_dict.get("name", None),
functions=list(parse_functions(target_dict)),
scaling_factor=float(target_dict.get(
"scaling_factor", "1")),
))
start_time = Timestamp.from_iso8601(request["range"]["from"])
end_time = Timestamp.from_iso8601(request["range"]["to"])
# Grafana gives inconsistent information here:
# intervalMs is very coarse grained
# maxDataPoints is not really the number of pixels, usually less
# interval = Timedelta.from_ms(request["intervalMs"])
interval = ((end_time - start_time) / request["maxDataPoints"]) * 2
results = await asyncio.gather(*[
target.get_response(app, start_time, end_time, interval)
for target in targets
])
rv = functools.reduce(operator.iconcat, results, [])
time_diff = timer() - time_begin
logger.log(
logging.DEBUG if time_diff < 1 else logging.INFO,
"get_history_data for {} targets took {} s",
len(targets),
time_diff,
)
return rv
def run_source(ssource):
ssource.declare_metrics(
{"dummy.time": {
"unit": "s",
"location": "localhost"
}})
try:
while True:
ssource.send("dummy.time", Timestamp.now(), time.time())
time.sleep(0.1)
except KeyboardInterrupt:
logger.info("stopping SynchronousSource")
ssource.stop()
def run(self):
assert self.import_begin is None
if not click.confirm(f'Please make sure the MetricQ db with the token '
f'{self.token}" is not running! Continue?'):
return
self.update_config()
self.create_bindings()
self.import_begin = Timestamp.now()
self.run_import()
if self.failed_imports:
print('The following metrics have failed to import:')
for metric in self.failed_imports:
print(f' - {metric.metricq_name}')
def convert(self, value: str, param: Optional[Parameter],
ctx: Optional[Context]) -> Any:
if value is None:
return None
elif isinstance(value, str):
try:
return Timestamp.from_iso8601(value)
except ValueError:
self.fail(
"expected an ISO-8601 timestamp (e.g. 2012-12-21T00:00:00Z)",
param=param,
ctx=ctx,
)
else:
return value
def real_run(self):
assert self._import_begin is None
self._confirm(f"Please make sure the MetricQ db with the token "
f'"{self._metricq_token}" is not running! Continue?')
self._update_config()
if not self._resume:
self._create_bindings()
self._import_begin = Timestamp.now()
self._run_import()
if self._failed_imports:
print("The following metrics have failed to import:")
for metric in self._failed_imports:
print(f" - {metric.metricq_name}")
class Ticker:
DEFAULT_DELTA = Timedelta.from_s(1)
DEFAULT_START = Timestamp(0)
def __init__(self, delta=DEFAULT_DELTA, start=DEFAULT_START):
self.delta = delta
self.start = start
self.now = self.start
def __next__(self):
now = self.now
self.now += self.delta
return now
def __iter__(self):
return self
def synchronous_source(server, token):
ssource = SynchronousSource(token=token, management_url=server)
ssource.declare_metrics({
"test.example.random": {
"unit": "s",
"description":
"a test metric that just contains random numbers in the range [0.0, 1.0)",
"rate": 10.0,
"location": "localhost",
}
})
try:
while True:
ssource.send("test.example.random", Timestamp.now(),
random.random())
time.sleep(0.1)
except KeyboardInterrupt:
logger.info("stopping SynchronousSource")
ssource.stop()
assert isclose(agg.mean_integral, VALUE)
assert isclose(agg.mean_sum, VALUE)
def test_timeaggregate_from_value_pair_non_monotonic(
timestamp: Timestamp, time_delta_10s: Timedelta):
later = timestamp + time_delta_10s
with pytest.raises(NonMonotonicTimestamps):
TimeAggregate.from_value_pair(timestamp_before=later,
timestamp=timestamp,
value=42.0)
@pytest.mark.parametrize(
("date_string", "expected"),
[
# Sanity check
("1970-01-01T00:00:00Z", Timestamp(0)),
# Parser supports sub-second digits
("1970-01-01T00:00:00.0Z", Timestamp(0)),
# Parser drops sub-microsecond digits
("1970-01-01T00:00:00.000001337Z", Timestamp(1000)),
# Timezones other that UTC are supported
("1970-01-01T00:00:00-01:00", Timestamp(
Timedelta.from_string("1h").ns)),
],
)
def test_timestamp_from_iso8601(date_string: str, expected: Timestamp):
assert Timestamp.from_iso8601(date_string) == expected
def test_timestamp_from_iso8601(date_string: str, expected: Timestamp):
assert Timestamp.from_iso8601(date_string) == expected
def test_timestamp_param():
value = "2021-05-02T00:00:00Z"
assert TIMESTAMP.convert(value, param=None,
ctx=None) == Timestamp.from_iso8601(value)
def timestamp():
return Timestamp.from_iso8601("2021-03-03T18:00:00Z")
def history_with_epoch_set():
history = StateTransitionHistory(None)
history.insert(Timestamp(0), State.OK)
return history