def testCheckpointCreated(chkFreqPatched, looper, txnPoolNodeSet, client1,
wallet1, client1Connected, reqs_for_checkpoint):
"""
After requests less than `CHK_FREQ`, there should be one checkpoint
on each replica. After `CHK_FREQ`, one checkpoint should become stable
"""
# Send one batch less so checkpoint is not created
sendReqsToNodesAndVerifySuffReplies(
looper, wallet1, client1,
reqs_for_checkpoint - (chkFreqPatched.Max3PCBatchSize), 1)
# Deliberately waiting so as to verify that not more than 1 checkpoint is
# created
looper.runFor(2)
chkChkpoints(txnPoolNodeSet, 1)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
chkFreqPatched.Max3PCBatchSize, 1)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
1,
0,
retryWait=1,
timeout=timeout))
def test_checkpoint_created(chkFreqPatched, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint):
"""
After requests less than `CHK_FREQ`, there should be one checkpoint
on each replica. After `CHK_FREQ`, one checkpoint should become stable
"""
# Send one batch less so checkpoint is not created
sdk_send_random_and_check(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint - (chkFreqPatched.Max3PCBatchSize))
# Deliberately waiting so as to verify that not more than 1 checkpoint is
# created
looper.runFor(2)
chkChkpoints(txnPoolNodeSet, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched.Max3PCBatchSize)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + 30
looper.runFor(timeout)
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
1,
0,
retryWait=1,
timeout=timeout))
def testCheckpointCreated(chkFreqPatched, looper, txnPoolNodeSet, client1,
wallet1, client1Connected):
"""
After requests less than `CHK_FREQ`, there should be one checkpoint
on each replica. After `CHK_FREQ`, one checkpoint should become stable
"""
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, CHK_FREQ-1, 1)
# Deliberately waiting so as to verify that not more than 1 checkpoint is
# created
looper.runFor(2)
chkChkpoints(txnPoolNodeSet, 1)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1))
def testCheckpointCreated(chkFreqPatched, looper, txnPoolNodeSet, client1,
wallet1, client1Connected):
"""
After requests less than `CHK_FREQ`, there should be one checkpoint
on each replica. After `CHK_FREQ`, one checkpoint should become stable
"""
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, CHK_FREQ - 1,
1)
# Deliberately waiting so as to verify that not more than 1 checkpoint is
# created
looper.runFor(2)
chkChkpoints(txnPoolNodeSet, 1)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1))
def test_checkpoint_created(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint):
"""
After requests less than `CHK_FREQ`, there should be one checkpoint
on each replica. After `CHK_FREQ`, one checkpoint should become stable
"""
# Send one batch less so checkpoint is not created
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint - (chkFreqPatched.Max3PCBatchSize))
# Deliberately waiting so as to verify that not more than 1 checkpoint is
# created
looper.runFor(2)
chkChkpoints(txnPoolNodeSet, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
chkFreqPatched.Max3PCBatchSize)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout))
def test_unordered_request_freed_on_replica_removal(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched,
view_change):
node = txnPoolNodeSet[0]
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, cDelay(delay=sys.maxsize)):
req = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
assert len(node.requests) == 1
forwardedToBefore = next(iter(node.requests.values())).forwardedTo
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert len(node.requests) == 1
forwardedToAfter = next(iter(node.requests.values())).forwardedTo
assert forwardedToAfter == forwardedToBefore - 1
chkChkpoints(txnPoolNodeSet, 0)
sdk_get_replies(looper, req)
chkChkpoints(txnPoolNodeSet, 1)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0))
assert len(node.requests) == 0
def test_unordered_request_freed_on_replica_removal(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched,
view_change):
node = txnPoolNodeSet[0]
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, cDelay(delay=sys.maxsize)):
req = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
assert len(node.requests) == 1
forwardedToBefore = next(iter(node.requests.values())).forwardedTo
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert len(node.requests) == 1
forwardedToAfter = next(iter(node.requests.values())).forwardedTo
assert forwardedToAfter == forwardedToBefore - 1
chkChkpoints(txnPoolNodeSet, 0)
sdk_get_replies(looper, req)
chkChkpoints(txnPoolNodeSet, 1)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0))
assert len(node.requests) == 0
def test_ordered_request_freed_on_replica_removal(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched, view_change):
node = txnPoolNodeSet[0]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
assert len(node.requests) == 1
forwardedToBefore = next(iter(node.requests.values())).forwardedTo
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert len(node.requests) == 1
forwardedToAfter = next(iter(node.requests.values())).forwardedTo
assert forwardedToAfter == forwardedToBefore - 1
chkChkpoints(txnPoolNodeSet, 1)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0))
assert len(node.requests) == 0
def test_ordered_request_freed_on_replica_removal(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched,
view_change):
node = txnPoolNodeSet[0]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
assert len(node.requests) == 1
forwardedToBefore = next(iter(node.requests.values())).forwardedTo
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert len(node.requests) == 1
forwardedToAfter = next(iter(node.requests.values())).forwardedTo
assert forwardedToAfter == forwardedToBefore - 1
chkChkpoints(txnPoolNodeSet, 1)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0))
assert len(node.requests) == 0