def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2) # From the steward send a request creating a user with None role sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3) # From the created user send a request creating another user. # Dynamic validation of this request must fail since a user with None role cannot create users. # However, the 3PC-batch with the sent request must be ordered. with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4) # Send more requests to cause checkpoint stabilization sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0) # Send more requests to cause new checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1)
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper, txnPoolNodeSet, client1, wallet1, client1Connected): """ Test checkpointing across views. This test checks that checkpointing and garbage collection works correctly no matter if view change happened before a checkpoint or after a checkpoint """ batch_size = 2 send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, batch_size*sent_batches, sent_batches) # Check that correct garbage collection happens non_gced_batch_count = (sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size*non_gced_batch_count, non_gced_batch_count, non_gced_batch_count, retryWait=1)) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # Check that after view change, proper clean up is done for node in txnPoolNodeSet: for r in node.replicas: assert not r.checkpoints # No stashed checkpoint for previous view assert not [view_no for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo] assert r._h == 0 assert r._lastPrePrepareSeqNo == 0 assert r.h == 0 assert r.H == r._h + chkFreqPatched.LOG_SIZE checkRequestCounts(txnPoolNodeSet, 0, 0, 0) # Even after view change, chekpointing works send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, batch_size*sent_batches, sent_batches) looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count, non_gced_batch_count, non_gced_batch_count, retryWait=1)) # Send more batches so one more checkpoint happens. This is done so that # when this test finishes, all requests are garbage collected and the # next run of this test (with next param) has the calculations correct more = CHK_FREQ - non_gced_batch_count send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, batch_size * more, more) looper.run(eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, 0, retryWait=1))
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test checkpointing across views. This test checks that checkpointing and garbage collection works correctly no matter if view change happened before a checkpoint or after a checkpoint """ batch_size = chkFreqPatched.Max3PCBatchSize sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * sent_batches, sent_batches) # Check that correct garbage collection happens non_gced_batch_count = (sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count, non_gced_batch_count, retryWait=1)) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # Check that after view change, proper clean up is done for node in txnPoolNodeSet: for r in node.replicas.values(): assert not r.checkpoints # No stashed checkpoint for previous view assert not [view_no for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo] assert r._h == 0 assert r._lastPrePrepareSeqNo == 0 assert r.h == 0 assert r.H == r._h + chkFreqPatched.LOG_SIZE checkRequestCounts(txnPoolNodeSet, 0, 0) # Even after view change, chekpointing works sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * sent_batches, sent_batches) looper.run(eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count, non_gced_batch_count, retryWait=1)) # Send more batches so one more checkpoint happens. This is done so that # when this test finishes, all requests are garbage collected and the # next run of this test (with next param) has the calculations correct more = CHK_FREQ - non_gced_batch_count sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * more, more) looper.run(eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, retryWait=1))
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper, txnPoolNodeSet, client1, wallet1, client1Connected, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) def send_and_wait_replies(num_reqs): return sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, num_reqs, fVal=0) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable reqs = send_and_wait_replies(reqs_for_checkpoint - max_batch_size) total_checkpoints = 1 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, retryWait=1, timeout=timeout)) chk_freq = chkFreqPatched.CHK_FREQ checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1) # Send some more requests to cause checkpoint stabilization send_and_wait_replies(max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed stable_checkpoint_id = 0 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, stable_checkpoint_id, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0, 0) # Send some more requests to cause new checkpoint send_and_wait_replies(reqs_for_checkpoint + 1) total_checkpoints = 2 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, stable_checkpoint_id, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable reqs = send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint - max_batch_size) total_checkpoints = 1 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, retryWait=1, timeout=timeout)) chk_freq = chkFreqPatched.CHK_FREQ checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1) # Send some more requests to cause checkpoint stabilization send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed stable_checkpoint_id = 0 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, stable_checkpoint_id, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0, 0) # Send some more requests to cause new checkpoint send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint + 1) total_checkpoints = 2 looper.run( eventually(chkChkpoints, txnPoolNodeSet, total_checkpoints, stable_checkpoint_id, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper, txnPoolNodeSet, client1, wallet1, client1Connected, reqs_for_checkpoint): max_batch_size = chkFreqPatched.Max3PCBatchSize chk_freq = chkFreqPatched.CHK_FREQ reqs = sendReqsToNodesAndVerifySuffReplies( looper, wallet1, client1, reqs_for_checkpoint - max_batch_size, 1) timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) looper.run( eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, len(reqs), chk_freq - 1, 1) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, max_batch_size, 1) looper.run( eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0, 0) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, reqs_for_checkpoint + 1, 1) looper.run( eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1, 1)
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test checkpointing across views. This test checks that checkpointing and garbage collection works correctly no matter if view change happened before a checkpoint or after a checkpoint """ batch_size = chkFreqPatched.Max3PCBatchSize sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * sent_batches, sent_batches) # Check that correct garbage collection happens non_gced_batch_count = ( sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches looper.run( eventually(checkRequestCounts, txnPoolNodeSet, batch_size * non_gced_batch_count, non_gced_batch_count, retryWait=1)) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # Check that after view change, proper clean up is done for node in txnPoolNodeSet: for r in node.replicas.values(): # Checkpoint was started after sending audit txn # assert not r.checkpoints # No stashed checkpoint for previous view assert not [ view_no for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo ] assert r._h == 0 # from audit txn assert r._lastPrePrepareSeqNo == 1 assert r.h == 0 assert r.H == r._h + chkFreqPatched.LOG_SIZE # All this manipulations because after view change we will send an empty batch for auditing checkRequestCounts(txnPoolNodeSet, 0, 1) if sent_batches > CHK_FREQ: expected_batch_count = sent_batches - CHK_FREQ + 1 additional_after_vc = 0 elif sent_batches == CHK_FREQ: expected_batch_count = 0 additional_after_vc = 0 sent_batches = CHK_FREQ - 1 else: expected_batch_count = sent_batches + 1 additional_after_vc = 1 # Even after view change, chekpointing works sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * sent_batches, sent_batches) looper.run( eventually(checkRequestCounts, txnPoolNodeSet, batch_size * (expected_batch_count - additional_after_vc), expected_batch_count, retryWait=1)) # Send more batches so one more checkpoint happens. This is done so that # when this test finishes, all requests are garbage collected and the # next run of this test (with next param) has the calculations correct more = CHK_FREQ - expected_batch_count sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, batch_size * more, more) looper.run( eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, retryWait=1))
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2) # From the steward send a request creating a user with None role sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3) # From the created user send a request creating another user. # Dynamic validation of this request must fail since a user with None role cannot create users. # However, the 3PC-batch with the sent request must be ordered. with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4) # Send more requests to cause checkpoint stabilization sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 5, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0) # Send more requests to cause new checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 10, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1)