def test_load_metrics_from_kv_store_can_load_all_values(storage):
events = generate_events(10)
step = timedelta(seconds=5)
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
expected_stats = MetricsStats(step)
for ev in events:
ts.value = ev.timestamp
metrics.store_event(ev.name, ev.value)
expected_stats.add(ev.timestamp, ev.name, ev.value)
stats = load_metrics_from_kv_store(storage, step=step)
assert stats == expected_stats
def test_equal_votes_dont_accumulate_when_added(instance_changes, tconf):
frm = "Node1"
view_no = 1
time_provider = MockTimestamp(0)
second_vote_time = 1
instance_changes = InstanceChanges(tconf, time_provider)
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_changes.add_vote(msg, frm)
time_provider.value = second_vote_time
instance_changes.add_vote(msg, frm)
assert instance_changes[view_no].voters[frm] == second_vote_time
assert len(instance_changes[view_no].voters) == 1
assert len(instance_changes) == 1
def test_timer_can_schedule_callback():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb = Callback()
timer.schedule(5, cb)
assert cb.call_count == 0
timer.service()
assert cb.call_count == 0
ts.value += 3
timer.service()
assert cb.call_count == 0
ts.value += 3
timer.service()
assert cb.call_count == 1
timer.service()
assert cb.call_count == 1
ts.value += 6
timer.service()
assert cb.call_count == 1
def test_load_metrics_from_kv_store_can_filter_values(storage):
events = generate_events(10)
step = timedelta(seconds=3)
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
expected_stats = MetricsStats(step)
timestamps = sorted(ev.timestamp for ev in events)
min_ts = timestamps[len(events) // 3]
max_ts = timestamps[2 * len(events) // 3]
for ev in events:
ts.value = ev.timestamp
metrics.store_event(ev.name, ev.value)
if min_ts <= ev.timestamp <= max_ts:
expected_stats.add(ev.timestamp, ev.name, ev.value)
stats = load_metrics_from_kv_store(storage, min_ts, max_ts, step)
assert stats == expected_stats
def test_timer_triggers_callback_on_time():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb = Callback()
timer.schedule(5, cb)
assert cb.call_count == 0
ts.value += 5
timer.service()
assert cb.call_count == 1
def test_timer_can_schedule_and_process_callback_twice():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb = Callback()
timer.schedule(3, cb)
timer.schedule(5, cb)
ts.value += 6
timer.service()
assert cb.call_count == 2
def test_old_ic_discard(instance_changes, tconf):
frm = "Node1"
view_no = 1
quorum = 2
time_provider = MockTimestamp(0)
instance_changes = InstanceChanges(tconf, time_provider)
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
time_provider.value = 0
instance_changes.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_changes.has_view(view_no)
instance_changes.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_changes.has_inst_chng_from(view_no, frm)
instance_changes.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_changes.has_quorum(view_no, quorum)
def test_timer_cancels_all_instances_of_callback():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb = Callback()
timer.schedule(5, cb)
timer.schedule(3, cb)
timer.cancel(cb)
ts.value += 6
timer.service()
assert cb.call_count == 0
def test_timer_can_schedule_different_callbacks_on_same_time_twice():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb1 = Callback()
cb2 = Callback()
timer.schedule(5, cb1)
timer.schedule(5, cb2)
ts.value += 6
timer.service()
assert cb1.call_count == 1
assert cb2.call_count == 1
def test_timer_can_schedule_and_simultaneously_process_different_callbacks():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb1 = Callback()
cb2 = Callback()
timer.schedule(5, cb1)
timer.schedule(3, cb2)
ts.value += 6
timer.service()
assert cb1.call_count == 1
assert cb2.call_count == 1
def test_kv_store_metrics_collector_store_all_data_in_order(
storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
events = generate_events(10)
for e in events:
ts.value = e.timestamp
metrics.store_event(e.name, e.value)
stored_events = [
KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()
]
# Check that all events are stored
assert len(stored_events) == len(events)
# Check that all events are stored in correct order
assert sorted(stored_events, key=lambda v: v.timestamp) == stored_events
# Check that all events stored were in source events
for ev in stored_events:
assert ev in events
# Check that all source events are in stored events
for ev in events:
assert ev in stored_events
def test_kv_store_metrics_collector_stores_properly_encoded_data(
storage: KeyValueStorage, value):
ts = MockTimestamp(gen_next_timestamp())
metrics = KvStoreMetricsCollector(storage, ts)
assert len([(k, v) for k, v in storage.iterator()]) == 0
id = gen_metrics_name()
event = MetricsEvent(ts.value, id, value)
encoded_key, encoded_value = KvStoreMetricsFormat.encode(event)
metrics.store_event(id, value)
stored_events = [(k, v) for k, v in storage.iterator()]
assert len(stored_events) == 1
assert stored_events[0][0] == encoded_key
assert stored_events[0][1] == encoded_value
def test_timer_cancel_callback_doesnt_crash_for_nonexistant_callback():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb = Callback()
# This shouldn't crash
timer.cancel(cb)
# Make sure that callback which was scheduled later is still called
timer.schedule(5, cb)
ts.value += 6
timer.service()
assert cb.call_count == 1
# And this still shouldn't crash
timer.cancel(cb)
def test_kv_store_metrics_collector_store_all_events_with_same_timestamp(
storage: KeyValueStorage):
ts = MockTimestamp()
metrics = KvStoreMetricsCollector(storage, ts)
values = [10, 2, 54, 2]
for v in values:
metrics.store_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, v)
events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]
# Check that all events are stored
assert len(events) == len(values)
# Check that all events are stored in correct order
assert sorted(events, key=lambda ev: ev.timestamp) == events
# Check that all events stored were in source events
for ev in events:
assert ev.value in values
def test_timer_can_cancel_callback_without_touching_other_callbacks():
ts = MockTimestamp(0)
timer = QueueTimer(ts)
cb1 = Callback()
cb2 = Callback()
cb3 = Callback()
timer.schedule(5, cb1)
timer.schedule(3, cb2)
timer.schedule(4, cb3)
timer.cancel(cb2)
ts.value += 6
timer.service()
assert cb1.call_count == 1
assert cb2.call_count == 0
assert cb3.call_count == 1
def test_too_old_messages_dont_count_towards_quorum(instance_changes, tconf):
frm1 = "Node1"
frm2 = "Node2"
view_no = 1
quorum = 2
time_provider = MockTimestamp(0)
instance_changes = InstanceChanges(tconf, time_provider)
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_changes.add_vote(msg, frm1)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2)
instance_changes.add_vote(msg, frm2)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2) + 1
assert not instance_changes.has_quorum(view_no, quorum)
assert instance_changes.has_view(view_no)
assert instance_changes[view_no].msg == msg
assert not instance_changes.has_inst_chng_from(view_no, frm1)
assert instance_changes.has_inst_chng_from(view_no, frm2)
def mock_timestamp():
return MockTimestamp(OBSOLETE_PP_TS)
def mock_timestamp():
return MockTimestamp()
def time_provider():
return MockTimestamp(0)
def mock_timestamp():
return MockTimestamp(OLDEST_TS)
