def test_kv_store_metrics_collector_stores_properly_encoded_data(storage: KeyValueStorage, value):
ts = MockTimestamp(gen_next_timestamp())
metrics = KvStoreMetricsCollector(storage, ts)
assert len([(k, v) for k, v in storage.iterator()]) == 0
id = gen_metrics_name()
event = MetricsEvent(ts.value, id, value)
encoded_key, encoded_value = KvStoreMetricsFormat.encode(event)
metrics.add_event(id, value)
stored_events = [(k, v) for k, v in storage.iterator()]
assert len(stored_events) == 1
assert stored_events[0][0] == encoded_key
assert stored_events[0][1] == encoded_value
def test_kv_store_metrics_collector_can_store_data(storage: KeyValueStorage):
ts = MockTimestamp(datetime(2018, month=7, day=24, hour=18, minute=12, second=35, microsecond=456000))
metrics = KvStoreMetricsCollector(storage, ts)
assert len([(k, v) for k, v in storage.iterator()]) == 0
id = MetricType.LOOPER_RUN_TIME_SPENT
value = 6.2
metrics.add_event(id, value)
result = [(k, v) for k, v in storage.iterator()]
assert len(result) == 1
k, v = result[0]
decoded_id, decoded_ts = decode_key(k)
assert decoded_id == id
assert decoded_ts == ts.value
assert decode_value(v) == value
def test_kv_store_metrics_collector_store_all_events_with_same_timestamp(storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
values = [10, 2, 54, 2]
for v in values:
metrics.add_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, v)
events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]
# Check that all events are stored
assert len(events) == len(values)
# Check that all events are stored in correct order
assert sorted(events, key=lambda ev: ev.timestamp) == events
# Check that all events stored were in source events
for ev in events:
assert ev.value in values
def load_metrics_from_kv_store(
storage: KeyValueStorage,
min_ts: datetime = None,
max_ts: datetime = None,
step: timedelta = timedelta(minutes=1)
) -> MetricsStats:
result = MetricsStats(step)
start = KvStoreMetricsFormat.encode_key(min_ts, 0) if min_ts else None
for k, v in storage.iterator(start=start):
ev = KvStoreMetricsFormat.decode(k, v)
if max_ts is not None and ev.timestamp > max_ts:
break
result.add(ev.timestamp, ev.name, ev.value)
return result
def load_metrics_from_kv_store(storage: KeyValueStorage,
min_ts: datetime = None,
max_ts: datetime = None,
step: timedelta = timedelta(minutes=1)) -> MetricsStats:
result = MetricsStats(step)
start = KvStoreMetricsFormat.encode_key(min_ts, 0) if min_ts else None
for k, v in storage.iterator(start=start):
ev = KvStoreMetricsFormat.decode(k, v)
if ev is None:
continue
if max_ts is not None and ev.timestamp > max_ts:
break
result.add(ev.timestamp, ev.name, ev.value)
return result
def test_kv_store_metrics_collector_store_all_data_in_order(
storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
events = generate_events(10)
for e in events:
ts.value = e.timestamp
metrics.add_event(e.name, e.value)
stored_events = [
KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()
]
# Check that all events are stored
assert len(stored_events) == len(events)
# Check that all events are stored in correct order
assert sorted(stored_events, key=lambda v: v.timestamp) == stored_events
# Check that all events stored were in source events
for ev in stored_events:
assert ev in events
def test_kv_store_metrics_collector_store_all_data_in_order(storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
events = [
(MetricType.CLIENT_STACK_MESSAGES_PROCESSED, 30),
(MetricType.LOOPER_RUN_TIME_SPENT, 1.2),
(MetricType.CLIENT_STACK_MESSAGES_PROCESSED, 163),
(MetricType.LOOPER_RUN_TIME_SPENT, 5.2),
(MetricType.CLIENT_STACK_MESSAGES_PROCESSED, 6),
(MetricType.LOOPER_RUN_TIME_SPENT, 0.3)
]
for id, value in events:
metrics.add_event(id, value)
ts.value += timedelta(seconds=0.1)
result = [(*decode_key(k), decode_value(v)) for k, v in storage.iterator()]
# Check that all events are stored
assert len(result) == 6
# Check that all events are stored in correct order
assert sorted(result, key=lambda v: (v[0], v[1])) == result
# Check that all events stored were in source events
for v in result:
assert (v[0], v[2]) in events