def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2)
# From the steward send a request creating a user with None role
sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3)
# From the created user send a request creating another user.
# Dynamic validation of this request must fail since a user with None role cannot create users.
# However, the 3PC-batch with the sent request must be ordered.
with pytest.raises(RequestRejectedException):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4)
# Send more requests to cause checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Send more requests to cause new checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1)
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper,
txnPoolNodeSet, client1, wallet1,
client1Connected):
"""
Test checkpointing across views.
This test checks that checkpointing and garbage collection works correctly
no matter if view change happened before a checkpoint or after a checkpoint
"""
batch_size = 2
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1,
batch_size*sent_batches,
sent_batches)
# Check that correct garbage collection happens
non_gced_batch_count = (sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches
looper.run(eventually(checkRequestCounts, txnPoolNodeSet,
batch_size*non_gced_batch_count, non_gced_batch_count,
non_gced_batch_count,
retryWait=1))
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# Check that after view change, proper clean up is done
for node in txnPoolNodeSet:
for r in node.replicas:
assert not r.checkpoints
# No stashed checkpoint for previous view
assert not [view_no for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo]
assert r._h == 0
assert r._lastPrePrepareSeqNo == 0
assert r.h == 0
assert r.H == r._h + chkFreqPatched.LOG_SIZE
checkRequestCounts(txnPoolNodeSet, 0, 0, 0)
# Even after view change, chekpointing works
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1,
batch_size*sent_batches,
sent_batches)
looper.run(eventually(checkRequestCounts, txnPoolNodeSet,
batch_size * non_gced_batch_count,
non_gced_batch_count,
non_gced_batch_count,
retryWait=1))
# Send more batches so one more checkpoint happens. This is done so that
# when this test finishes, all requests are garbage collected and the
# next run of this test (with next param) has the calculations correct
more = CHK_FREQ - non_gced_batch_count
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1,
batch_size * more,
more)
looper.run(eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, 0,
retryWait=1))
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Test checkpointing across views.
This test checks that checkpointing and garbage collection works correctly
no matter if view change happened before a checkpoint or after a checkpoint
"""
batch_size = chkFreqPatched.Max3PCBatchSize
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
batch_size * sent_batches, sent_batches)
# Check that correct garbage collection happens
non_gced_batch_count = (sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches
looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count,
non_gced_batch_count, retryWait=1))
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# Check that after view change, proper clean up is done
for node in txnPoolNodeSet:
for r in node.replicas.values():
assert not r.checkpoints
# No stashed checkpoint for previous view
assert not [view_no for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo]
assert r._h == 0
assert r._lastPrePrepareSeqNo == 0
assert r.h == 0
assert r.H == r._h + chkFreqPatched.LOG_SIZE
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Even after view change, chekpointing works
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
batch_size * sent_batches, sent_batches)
looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count,
non_gced_batch_count, retryWait=1))
# Send more batches so one more checkpoint happens. This is done so that
# when this test finishes, all requests are garbage collected and the
# next run of this test (with next param) has the calculations correct
more = CHK_FREQ - non_gced_batch_count
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
batch_size * more, more)
looper.run(eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, retryWait=1))
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper,
txnPoolNodeSet, client1,
wallet1, client1Connected,
reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
def send_and_wait_replies(num_reqs):
return sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
num_reqs,
fVal=0)
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
reqs = send_and_wait_replies(reqs_for_checkpoint - max_batch_size)
total_checkpoints = 1
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
retryWait=1,
timeout=timeout))
chk_freq = chkFreqPatched.CHK_FREQ
checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1)
# Send some more requests to cause checkpoint stabilization
send_and_wait_replies(max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
stable_checkpoint_id = 0
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
stable_checkpoint_id,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0, 0)
# Send some more requests to cause new checkpoint
send_and_wait_replies(reqs_for_checkpoint + 1)
total_checkpoints = 2
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
stable_checkpoint_id,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
reqs = send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
reqs_for_checkpoint - max_batch_size)
total_checkpoints = 1
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
retryWait=1,
timeout=timeout))
chk_freq = chkFreqPatched.CHK_FREQ
checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1)
# Send some more requests to cause checkpoint stabilization
send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
stable_checkpoint_id = 0
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
stable_checkpoint_id,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0, 0)
# Send some more requests to cause new checkpoint
send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint + 1)
total_checkpoints = 2
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
total_checkpoints,
stable_checkpoint_id,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper,
txnPoolNodeSet, client1,
wallet1, client1Connected,
reqs_for_checkpoint):
max_batch_size = chkFreqPatched.Max3PCBatchSize
chk_freq = chkFreqPatched.CHK_FREQ
reqs = sendReqsToNodesAndVerifySuffReplies(
looper, wallet1, client1, reqs_for_checkpoint - max_batch_size, 1)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
1,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
max_batch_size, 1)
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
1,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0, 0)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
reqs_for_checkpoint + 1, 1)
looper.run(
eventually(chkChkpoints,
txnPoolNodeSet,
2,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper,
txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
"""
Test checkpointing across views.
This test checks that checkpointing and garbage collection works correctly
no matter if view change happened before a checkpoint or after a checkpoint
"""
batch_size = chkFreqPatched.Max3PCBatchSize
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
batch_size * sent_batches,
sent_batches)
# Check that correct garbage collection happens
non_gced_batch_count = (
sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches
looper.run(
eventually(checkRequestCounts,
txnPoolNodeSet,
batch_size * non_gced_batch_count,
non_gced_batch_count,
retryWait=1))
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# Check that after view change, proper clean up is done
for node in txnPoolNodeSet:
for r in node.replicas.values():
# Checkpoint was started after sending audit txn
# assert not r.checkpoints
# No stashed checkpoint for previous view
assert not [
view_no
for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo
]
assert r._h == 0
# from audit txn
assert r._lastPrePrepareSeqNo == 1
assert r.h == 0
assert r.H == r._h + chkFreqPatched.LOG_SIZE
# All this manipulations because after view change we will send an empty batch for auditing
checkRequestCounts(txnPoolNodeSet, 0, 1)
if sent_batches > CHK_FREQ:
expected_batch_count = sent_batches - CHK_FREQ + 1
additional_after_vc = 0
elif sent_batches == CHK_FREQ:
expected_batch_count = 0
additional_after_vc = 0
sent_batches = CHK_FREQ - 1
else:
expected_batch_count = sent_batches + 1
additional_after_vc = 1
# Even after view change, chekpointing works
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
batch_size * sent_batches,
sent_batches)
looper.run(
eventually(checkRequestCounts,
txnPoolNodeSet,
batch_size * (expected_batch_count - additional_after_vc),
expected_batch_count,
retryWait=1))
# Send more batches so one more checkpoint happens. This is done so that
# when this test finishes, all requests are garbage collected and the
# next run of this test (with next param) has the calculations correct
more = CHK_FREQ - expected_batch_count
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
batch_size * more, more)
looper.run(
eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, retryWait=1))
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 2 * max_batch_size)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2)
# From the steward send a request creating a user with None role
sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle,
sdk_wallet_steward)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3)
# From the created user send a request creating another user.
# Dynamic validation of this request must fail since a user with None role cannot create users.
# However, the 3PC-batch with the sent request must be ordered.
with pytest.raises(RequestRejectedException):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4)
# Send more requests to cause checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
5,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Send more requests to cause new checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, reqs_for_checkpoint + 1)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
10,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1)
