async def test_delay_with_soft_limit_reached_counter(self, bft_network):
"""
1. Start all replicas except a non-primary
2. Expected result: The cluster should work in slow path
3. Add a delay of 500ms in network
4. Start the stopped non-primary replica
5. Expected result: Increment in soft_limit_reached_counter
6. Remove the delay from network
"""
initial_primary = 0
skvbc = kvbc.SimpleKVBCProtocol(bft_network)
non_primary_replica = random.choice(
bft_network.all_replicas(without={initial_primary}))
bft_network.start_replicas(bft_network.all_replicas(without={non_primary_replica}))
await bft_network.wait_for_slow_path_to_be_prevalent(
run_ops=lambda: skvbc.run_concurrent_ops(num_ops=20, write_weight=1), threshold=20)
with ntc.NetworkTrafficControl() as tc:
#delay added to loopback interface
tc.put_loop_back_interface_delay(500)
bft_network.start_replica(non_primary_replica)
await skvbc.run_concurrent_ops(100, 1)
with log.start_action(action_type="get_soft_limit_reached_counter") as action:
soft_limit_cnt = await bft_network.get_metric(non_primary_replica, bft_network, "Counters",
"soft_limit_reached_counter", "time_service")
action.log(
message_type=f'soft_limit_reached_counter #{soft_limit_cnt}')
self.assertGreater(soft_limit_cnt, 0)
async def test_state_transfer_with_internal_cycle(self,
bft_network,
exchange_keys=True):
"""
state transfer starts and destination replica(fetcher) is
blocked during getMissingBlocks state, a bunch of data is added
to the rest of the cluster using multiple clients. Then destination
replica is unblocked and state transfer resume . After one cycle
of state transfer is done replica goes into another internal(
from getReserved page state to getCheckpointSummary State)
cycle to fetch remaining blocks which were added during first cycle.
"""
skvbc = kvbc.SimpleKVBCProtocol(bft_network)
stale_node = random.choice(bft_network.all_replicas(without={0}))
await skvbc.start_replicas_and_write_with_multiple_clients(
stale_nodes={stale_node},
write_run_duration=30,
persistency_enabled=False)
with ntc.NetworkTrafficControl() as tc:
#delay added to loopback interface to capture the stale replica in GettingMissingBlocks state
tc.put_loop_back_interface_delay(200)
bft_network.start_replica(stale_node)
with trio.fail_after(30):
while (True):
fetchingState = await bft_network.get_metric(
stale_node, bft_network, "Statuses", "fetching_state",
"bc_state_transfer")
if (fetchingState == "GettingMissingBlocks"):
break
await trio.sleep(0.1)
await skvbc.send_concurrent_ops_with_isolated_replica(
isolated_replica={stale_node}, run_duration=30)
await bft_network.wait_for_state_transfer_to_stop(0, stale_node)
await bft_network.wait_for_replicas_rvt_root_values_to_be_in_sync(
bft_network.all_replicas())
await bft_network.force_quorum_including_replica(stale_node)
await skvbc.assert_successful_put_get()