def test_kv_store_decode_restores_encoded_event():
event = MetricsEvent(gen_next_timestamp(), gen_metrics_name(), 4.2)
k, v = KvStoreMetricsFormat.encode(event)
decoded_event = KvStoreMetricsFormat.decode(k, v)
assert event == decoded_event
def test_kv_store_encode_generate_different_keys_for_different_seq_no():
event = MetricsEvent(gen_next_timestamp(), gen_metrics_name(), 4.2)
k1, v1 = KvStoreMetricsFormat.encode(event, 1)
k2, v2 = KvStoreMetricsFormat.encode(event, 2)
assert k1 != k2
assert v1 == v2
def load_metrics_from_kv_store(
storage: KeyValueStorage,
min_ts: datetime = None,
max_ts: datetime = None,
step: timedelta = timedelta(minutes=1)
) -> MetricsStats:
result = MetricsStats(step)
start = KvStoreMetricsFormat.encode_key(min_ts, 0) if min_ts else None
for k, v in storage.iterator(start=start):
ev = KvStoreMetricsFormat.decode(k, v)
if max_ts is not None and ev.timestamp > max_ts:
break
result.add(ev.timestamp, ev.name, ev.value)
return result
def load_metrics_from_kv_store(storage: KeyValueStorage,
min_ts: datetime = None,
max_ts: datetime = None,
step: timedelta = timedelta(minutes=1)) -> MetricsStats:
result = MetricsStats(step)
start = KvStoreMetricsFormat.encode_key(min_ts, 0) if min_ts else None
for k, v in storage.iterator(start=start):
ev = KvStoreMetricsFormat.decode(k, v)
if ev is None:
continue
if max_ts is not None and ev.timestamp > max_ts:
break
result.add(ev.timestamp, ev.name, ev.value)
return result
def check_metrics_data(storage):
events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]
# Check that metrics are actually written
assert len(events) > 0
# Check that all events are stored in correct order
assert sorted(events, key=lambda v: v.timestamp) == events
# Check that all event types happened during test
metric_names = {ev.name for ev in events}
for t in MetricsName:
assert t in metric_names
def test_kv_store_metrics_collector_stores_properly_encoded_data(storage: KeyValueStorage, value):
ts = MockTimestamp(gen_next_timestamp())
metrics = KvStoreMetricsCollector(storage, ts)
assert len([(k, v) for k, v in storage.iterator()]) == 0
id = gen_metrics_name()
event = MetricsEvent(ts.value, id, value)
encoded_key, encoded_value = KvStoreMetricsFormat.encode(event)
metrics.add_event(id, value)
stored_events = [(k, v) for k, v in storage.iterator()]
assert len(stored_events) == 1
assert stored_events[0][0] == encoded_key
assert stored_events[0][1] == encoded_value
def test_kv_store_metrics_collector_store_all_events_with_same_timestamp(storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
values = [10, 2, 54, 2]
for v in values:
metrics.add_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, v)
events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]
# Check that all events are stored
assert len(events) == len(values)
# Check that all events are stored in correct order
assert sorted(events, key=lambda ev: ev.timestamp) == events
# Check that all events stored were in source events
for ev in events:
assert ev.value in values
def test_kv_store_metrics_collector_store_all_data_in_order(
storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
events = generate_events(10)
for e in events:
ts.value = e.timestamp
metrics.add_event(e.name, e.value)
stored_events = [
KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()
]
# Check that all events are stored
assert len(stored_events) == len(events)
# Check that all events are stored in correct order
assert sorted(stored_events, key=lambda v: v.timestamp) == stored_events
# Check that all events stored were in source events
for ev in stored_events:
assert ev in events
def test_kv_store_metrics_config(looper, txnPoolNodeSet, tdir, tconf,
sdk_pool_handle, sdk_wallet_client):
total_time = 1.5 * tconf.PerfCheckFreq
total_iters = 5
iter_time = total_time / total_iters
for _ in range(total_iters):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 15)
looper.runFor(iter_time)
for node in txnPoolNodeSet:
storage = initKeyValueStorage(tconf.METRICS_KV_STORAGE,
node.dataLocation,
tconf.METRICS_KV_DB_NAME,
read_only=True)
events = [
KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()
]
# Check that metrics are actually written
assert len(events) > 0
# Check that all events are stored in correct order
assert sorted(events, key=lambda v: v.timestamp) == events
# We don't expect some events in this test
unexpected_events = {
MetricsName.CATCHUP_TXNS_SENT,
MetricsName.CATCHUP_TXNS_RECEIVED,
MetricsName.GC_GEN2_TIME,
MetricsName.GC_UNCOLLECTABLE_OBJECTS,
MetricsName.GC_GEN2_COLLECTED_OBJECTS,
MetricsName.PROCESS_CHECKPOINT_TIME,
MetricsName.SEND_CHECKPOINT_TIME,
MetricsName.BACKUP_PROCESS_CHECKPOINT_TIME,
MetricsName.BACKUP_SEND_CHECKPOINT_TIME,
MetricsName.PROCESS_CONSISTENCY_PROOF_TIME,
MetricsName.PROCESS_CATCHUP_REQ_TIME,
MetricsName.PROCESS_CATCHUP_REP_TIME,
MetricsName.NODE_CHECK_NODE_REQUEST_SPIKE,
MetricsName.NODE_SEND_REJECT_TIME,
# Obsolete metrics
MetricsName.DESERIALIZE_DURING_UNPACK_TIME,
# TODO: reduce monitor window so these events are also captured
MetricsName.MONITOR_AVG_THROUGHPUT,
MetricsName.BACKUP_MONITOR_AVG_THROUGHPUT,
MetricsName.MONITOR_AVG_LATENCY,
MetricsName.BACKUP_MONITOR_AVG_LATENCY,
# Temporary metrics
MetricsName.STORAGE_IDR_CACHE_READERS,
MetricsName.STORAGE_IDR_CACHE_TABLES_NUM,
MetricsName.STORAGE_IDR_CACHE_TABLES_SIZE,
MetricsName.STORAGE_ATTRIBUTE_STORE_READERS,
MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_NUM,
MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_SIZE
}
# Don't expect some metrics from master primary
if node.master_replica.isPrimary:
unexpected_events.add(MetricsName.PROCESS_PREPREPARE_TIME)
unexpected_events.add(MetricsName.SEND_PREPARE_TIME)
else:
unexpected_events.add(MetricsName.SEND_PREPREPARE_TIME)
unexpected_events.add(MetricsName.CREATE_3PC_BATCH_TIME)
unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)
# Don't expect some metrics from backup primary
assert node.replicas.num_replicas == 2
if node.replicas[1].isPrimary:
unexpected_events.add(MetricsName.BACKUP_PROCESS_PREPREPARE_TIME)
unexpected_events.add(MetricsName.BACKUP_SEND_PREPARE_TIME)
else:
unexpected_events.add(MetricsName.BACKUP_SEND_PREPREPARE_TIME)
unexpected_events.add(MetricsName.BACKUP_CREATE_3PC_BATCH_TIME)
unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)
if not node.primaryDecider:
unexpected_events.add(MetricsName.PRIMARY_DECIDER_ACTION_QUEUE)
unexpected_events.add(MetricsName.PRIMARY_DECIDER_AQ_STASH)
unexpected_events.add(
MetricsName.PRIMARY_DECIDER_REPEATING_ACTIONS)
unexpected_events.add(MetricsName.PRIMARY_DECIDER_SCHEDULED)
unexpected_events.add(MetricsName.PRIMARY_DECIDER_INBOX)
unexpected_events.add(MetricsName.PRIMARY_DECIDER_OUTBOX)
# Check that all event types happened during test
metric_names = {ev.name for ev in events}
for t in MetricsName:
if t in unexpected_events or t > TMP_METRIC:
continue
assert t in metric_names