def test_apply_data(fake_node, fake_node_observer, txnPoolNodeSet, observed_data_msgs): ''' - Create a Node (do not add it to the pool) with Observer. - Send txns and get ObservedData msgs from Validators. - Apply the ObservedData masgs by the Observer and make sure that it becomes synced with the pool. ''' # check that Observer is not synced with the pool checkNodeDataForInequality(fake_node, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc', 'check_seqno_db']) # emulate sending of ObserverData from each Node for observed_data_msg in observed_data_msgs: for node in txnPoolNodeSet: fake_node_observer.apply_data(observed_data_msg, node.name) # check that Observer is synced with the pool checkNodeDataForEquality(fake_node, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc', 'check_primaries', 'check_last_ordered_3pc_backup'])
def test_no_preprepare_requested(looper, txnPoolNodeSet, client1, wallet1, client1Connected, teardown): """ Node missing Propagates hence request not finalised, hence stashes PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE """ slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet) slow_node.nodeIbStasher.delay(ppgDelay(20)) slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ])) old_count_resp = count_requested_preprepare_resp(slow_node) send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 4, 2) # The slow node is behind checkNodeDataForInequality(slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp slow_node.nodeIbStasher.reset_delays_and_process_delayeds() # The slow node has processed all requests waitNodeDataEquality(looper, slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_catch_up_after_demoted(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): logger.info( "1. add a new node after sending some txns and check that catch-up " "is done (the new node is up to date)") looper, newNode, client, wallet, newStewardClient, \ newStewardWallet = nodeSetWithNodeAddedAfterSomeTxns waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:4]) logger.info("2. turn the new node off (demote)") node_data = {ALIAS: newNode.name, SERVICES: []} updateNodeData(looper, newStewardClient, newStewardWallet, newNode, node_data) logger.info("3. send more requests, " "so that the new node's state is outdated") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) checkNodeDataForInequality(newNode, *txnPoolNodeSet[:-1]) logger.info("4. turn the new node on") node_data = {ALIAS: newNode.name, SERVICES: [VALIDATOR]} updateNodeData(looper, newStewardClient, newStewardWallet, newNode, node_data) logger.info("5. make sure catch-up is done " "(the new node is up to date again)") waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) logger.info("6. send more requests and make sure " "that the new node participates in processing them") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 10) waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1])
def test_apply_data(fake_node, fake_node_observer, txnPoolNodeSet, observed_data_msgs): ''' - Create a Node (do not add it to the pool) with Observer. - Send txns and get ObservedData msgs from Validators. - Apply the ObservedData masgs by the Observer and make sure that it becomes synced with the pool. ''' # check that Observer is not synced with the pool checkNodeDataForInequality(fake_node, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc', 'check_seqno_db']) # emulate sending of ObserverData from each Node for observed_data_msg in observed_data_msgs: for node in txnPoolNodeSet: fake_node_observer.apply_data(observed_data_msg, node.name) # check that Observer is synced with the pool checkNodeDataForEquality(fake_node, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc'])
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ Node missing Propagates hence request not finalised, hence stashes PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE """ slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet) slow_node.nodeIbStasher.delay(ppgDelay()) slow_node.clientIbStasher.delay(req_delay()) slow_node.nodeIbStasher.delay(msg_rep_delay(1000, [ PROPAGATE, ])) old_count_resp = count_requested_preprepare_resp(slow_node) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=4, num_batches=2) # The slow node is behind checkNodeDataForInequality(slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp slow_node.nodeIbStasher.reset_delays_and_process_delayeds() # The slow node has processed all requests waitNodeDataEquality(looper, slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ Node missing Propagates hence request not finalised, hence stashes PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE """ slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet) slow_node.nodeIbStasher.delay(ppgDelay(20)) slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ])) old_count_resp = count_requested_preprepare_resp(slow_node) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=4, num_batches=2) # The slow node is behind checkNodeDataForInequality(slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp slow_node.nodeIbStasher.reset_delays_and_process_delayeds() # The slow node has processed all requests waitNodeDataEquality(looper, slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_lag_size_for_catchup(looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Verifies that if the stored own checkpoints have aligned bounds then the master replica lag which makes the node perform catch-up is Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed received checkpoints. """ slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node other_nodes = [n for n in txnPoolNodeSet if n != slow_node] # The master replica of the slow node stops to receive 3PC-messages slow_node.master_replica.threePhaseRouter.extend(( (PrePrepare, lambda *x, **y: None), (Prepare, lambda *x, **y: None), (Commit, lambda *x, **y: None), )) completed_catchups_before_reqs = get_number_of_completed_catchups( slow_node) # Send requests for the slow node's master replica to get # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP quorumed stashed checkpoints # from others send_reqs_batches_and_get_suff_replies( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP * reqs_for_checkpoint) # Give time for the slow node to catch up if it is going to do it looper.runFor( waits.expectedPoolConsistencyProof(len(txnPoolNodeSet)) + waits.expectedPoolCatchupTime(len(txnPoolNodeSet))) checkNodeDataForInequality(slow_node, *other_nodes) # Verify that the slow node has not caught up assert get_number_of_completed_catchups( slow_node) == completed_catchups_before_reqs # Send more requests for the slow node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint) waitNodeDataEquality(looper, slow_node, *other_nodes, exclude_from_check=['check_last_ordered_3pc_backup']) # Verify that the slow node has caught up assert get_number_of_completed_catchups( slow_node) > completed_catchups_before_reqs
def test_lag_size_for_catchup( looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Verifies that if the stored own checkpoints have aligned bounds then the master replica lag which makes the node perform catch-up is Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed received checkpoints. """ slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node other_nodes = [n for n in txnPoolNodeSet if n != slow_node] # The master replica of the slow node stops to receive 3PC-messages slow_node.master_replica.threePhaseRouter.extend( ( (PrePrepare, lambda *x, **y: None), (Prepare, lambda *x, **y: None), (Commit, lambda *x, **y: None), ) ) completed_catchups_before_reqs = get_number_of_completed_catchups(slow_node) # Send requests for the slow node's master replica to get # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP quorumed stashed checkpoints # from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP * reqs_for_checkpoint) # Give time for the slow node to catch up if it is going to do it looper.runFor(waits.expectedPoolConsistencyProof(len(txnPoolNodeSet)) + waits.expectedPoolCatchupTime(len(txnPoolNodeSet))) checkNodeDataForInequality(slow_node, *other_nodes) # Verify that the slow node has not caught up assert get_number_of_completed_catchups(slow_node) == completed_catchups_before_reqs # Send more requests for the slow node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint) waitNodeDataEquality(looper, slow_node, *other_nodes) # Verify that the slow node has caught up assert get_number_of_completed_catchups(slow_node) > completed_catchups_before_reqs
def test_catch_up_after_demoted(txnPoolNodeSet, sdk_node_set_with_node_added_after_some_txns, sdk_wallet_client): logger.info( "1. add a new node after sending some txns and check that catch-up " "is done (the new node is up to date)") looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ sdk_node_set_with_node_added_after_some_txns waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) logger.info("2. turn the new node off (demote)") node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet_handle, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, services=[]) logger.info("3. send more requests, " "so that the new node's state is outdated") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) checkNodeDataForInequality(new_node, *txnPoolNodeSet[:-1]) logger.info("4. turn the new node on") sdk_send_update_node(looper, new_steward_wallet_handle, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, services=[VALIDATOR]) logger.info("5. make sure catch-up is done " "(the new node is up to date again)") waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.info("6. send more requests and make sure " "that the new node participates in processing them") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet_handle, 10) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def test_catch_up_after_demoted( txnPoolNodeSet, sdk_node_set_with_node_added_after_some_txns, sdk_wallet_client): logger.info( "1. add a new node after sending some txns and check that catch-up " "is done (the new node is up to date)") looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ sdk_node_set_with_node_added_after_some_txns waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) logger.info("2. turn the new node off (demote)") node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet_handle, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, services=[]) logger.info("3. send more requests, " "so that the new node's state is outdated") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) checkNodeDataForInequality(new_node, *txnPoolNodeSet[:-1]) logger.info("4. turn the new node on") sdk_send_update_node(looper, new_steward_wallet_handle, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, services=[VALIDATOR]) logger.info("5. make sure catch-up is done " "(the new node is up to date again)") waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.info("6. send more requests and make sure " "that the new node participates in processing them") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet_handle, 10) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def test_non_primary_recvs_3phase_message_outside_watermarks( chkFreqPatched, reqs_for_logsize, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ A node is slow in receiving PRE-PREPAREs and PREPAREs. A lot of requests are sent and the slow node has started receiving COMMITs outside of its watermarks and so stashes them. Also this node is slow in receiving CHECKPOINTs. So a catch-up does not occur on it. Then the slow node eventually receives the sent PRE-PREPAREs and PREPAREs and so orders the 3PC-batches between its watermarks. The other nodes discard the COMMITs from the slow node since they have already achieved stable checkpoints for these COMMITs. After that the slow node eventually receives the sent CHECKPOINTs from the other nodes and so stabilizes own completed checkpoints and updates its watermarks. A catch-up is not triggered because no received checkpoints are stashed. Since now the watermarks have been updated, the slow node processes 3PC-messages stashed earlier and its ledger becomes equal to the ledgers of the other nodes. """ backupInstId = 1 npr = getNonPrimaryReplicas(txnPoolNodeSet, backupInstId) slowReplica = npr[0] slowNode = slowReplica.node slowNode.nodeIbStasher.delay(ppDelay(300, backupInstId)) slowNode.nodeIbStasher.delay(pDelay(300, backupInstId)) slowNode.nodeIbStasher.delay(chk_delay(300)) initialDomainLedgerSize = slowNode.domainLedger.size oldStashCount = slowReplica.stasher.num_stashed_watermarks slowReplica.H = LOG_SIZE # 1. Send requests more than fit between the watermarks on the slow node sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_logsize + 2) # Verify that the slow node stashes the batches outside of its watermarks newStashCount = slowReplica.stasher.num_stashed_watermarks assert newStashCount > oldStashCount oldDiscardCounts = discardCounts([n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode], 'achieved stable checkpoint') # 2. Deliver the sent PREPREPAREs and PREPAREs to the slow node slowNode.nodeIbStasher.reset_delays_and_process_delayeds(PREPREPARE, PREPARE) # Verify that the slow node orders the 3PC-batches between its watermarks # but no more. looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))) checkNodeDataForInequality(slowNode, *[n for n in txnPoolNodeSet if n != slowNode]) assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize # Also verify that the other nodes discard the COMMITs from the slow node # since they have already achieved stable checkpoints for these COMMITs. counts = discardCounts( [n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode], 'achieved stable checkpoint') for nm, count in counts.items(): assert count > oldDiscardCounts[nm] oldCatchupTimes = slowNode.spylog.count(Node.start_catchup) # 3. Deliver the sent CHECKPOINTs to the slow node slowNode.nodeIbStasher.reset_delays_and_process_delayeds(CHECKPOINT) # Verify that the slow node processes 3PC-messages stashed earlier and its # ledger becomes equal to the ledgers of the other nodes while a catch-up # is not made. waitNodeDataEquality(looper, slowNode, *[n for n in txnPoolNodeSet if n != slowNode]) assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize + 2 newCatchupTimes = slowNode.spylog.count(Node.start_catchup) assert newCatchupTimes == oldCatchupTimes
def test_complete_short_checkpoint_not_included_in_lag_for_catchup( looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tdir, tconf, allPluginsPath): """ Verifies that if the first stored own checkpoint has a not aligned lower bound (this means that it was started after a catch-up), is complete and there is a quorumed stashed checkpoint from other replicas with the same end then this stashed checkpoint is not included into the lag for a catch-up, i.e. in such a case the lag which makes the node perform catch-up is Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 2 quorumed stashed received checkpoints. """ max_batch_size = chkFreqPatched.Max3PCBatchSize _, new_node = sdk_add_new_steward_and_node( looper, sdk_pool_handle, sdk_wallet_steward, 'EpsilonSteward', 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Epsilon did not participate in ordering of the batch with EpsilonSteward # NYM transaction and the batch with Epsilon NODE transaction. # Epsilon got these transactions via catch-up. # To complete the first checkpoint send requests for 1 checkpoint minus # 2 3PC-batches (since there are already 2 3PC-batches in the first # checkpoint : with EpsilonSteward NYM transaction and with Epsilon NODE # transaction). This checkpoint has a not aligned lower bound # on the new node replicas so it will not be stabilized on them. send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint - 2 * max_batch_size) # The master replica of the new node stops to receive 3PC-messages new_node.master_replica.threePhaseRouter.extend( ( (PrePrepare, lambda *x, **y: None), (Prepare, lambda *x, **y: None), (Commit, lambda *x, **y: None), ) ) completed_catchups_before_reqs = get_number_of_completed_catchups(new_node) # Send requests for the new node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP * reqs_for_checkpoint) # Give time for the new node to catch up if it is going to do it looper.runFor(waits.expectedPoolConsistencyProof(len(txnPoolNodeSet)) + waits.expectedPoolCatchupTime(len(txnPoolNodeSet))) checkNodeDataForInequality(new_node, *txnPoolNodeSet[:-1]) # Verify that the new node has not caught up assert get_number_of_completed_catchups(new_node) == completed_catchups_before_reqs # Send more requests for the new node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 2 quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Verify that the new node has caught up assert get_number_of_completed_catchups(new_node) > completed_catchups_before_reqs
def test_non_primary_recvs_3phase_message_outside_watermarks( chkFreqPatched, reqs_for_logsize, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ A node is slow in receiving PRE-PREPAREs and PREPAREs. A lot of requests are sent and the slow node has started receiving COMMITs outside of its watermarks and so stashes them. Also this node is slow in receiving CHECKPOINTs. So a catch-up does not occur on it. Then the slow node eventually receives the sent PRE-PREPAREs and PREPAREs and so orders the 3PC-batches between its watermarks. The other nodes discard the COMMITs from the slow node since they have already achieved stable checkpoints for these COMMITs. After that the slow node eventually receives the sent CHECKPOINTs from the other nodes and so stabilizes own completed checkpoints and updates its watermarks. A catch-up is not triggered because no received checkpoints are stashed. Since now the watermarks have been updated, the slow node processes 3PC-messages stashed earlier and its ledger becomes equal to the ledgers of the other nodes. """ backupInstId = 1 npr = getNonPrimaryReplicas(txnPoolNodeSet, backupInstId) slowReplica = npr[0] slowNode = slowReplica.node slowNode.nodeIbStasher.delay(ppDelay(300, backupInstId)) slowNode.nodeIbStasher.delay(pDelay(300, backupInstId)) slowNode.nodeIbStasher.delay(chk_delay(300)) initialDomainLedgerSize = slowNode.domainLedger.size oldStashCount = slowReplica.spylog.count(TestReplica.stashOutsideWatermarks.__name__) # 1. Send requests more than fit between the watermarks on the slow node sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_logsize + 2) # Verify that the slow node stashes the batches outside of its watermarks newStashCount = slowReplica.spylog.count(TestReplica.stashOutsideWatermarks.__name__) assert newStashCount > oldStashCount oldDiscardCounts = discardCounts([n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode], 'achieved stable checkpoint') # 2. Deliver the sent PREPREPAREs and PREPAREs to the slow node slowNode.nodeIbStasher.reset_delays_and_process_delayeds(PREPREPARE, PREPARE) # Verify that the slow node orders the 3PC-batches between its watermarks # but no more. looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))) checkNodeDataForInequality(slowNode, *[n for n in txnPoolNodeSet if n != slowNode]) assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize # Also verify that the other nodes discard the COMMITs from the slow node # since they have already achieved stable checkpoints for these COMMITs. counts = discardCounts( [n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode], 'achieved stable checkpoint') for nm, count in counts.items(): assert count > oldDiscardCounts[nm] oldCatchupTimes = slowNode.spylog.count(Node.start_catchup) # 3. Deliver the sent CHECKPOINTs to the slow node slowNode.nodeIbStasher.reset_delays_and_process_delayeds(CHECKPOINT) # Verify that the slow node processes 3PC-messages stashed earlier and its # ledger becomes equal to the ledgers of the other nodes while a catch-up # is not made. waitNodeDataEquality(looper, slowNode, *[n for n in txnPoolNodeSet if n != slowNode]) assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize + 2 newCatchupTimes = slowNode.spylog.count(Node.start_catchup) assert newCatchupTimes == oldCatchupTimes
def test_incomplete_short_checkpoint_included_in_lag_for_catchup( looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tdir, tconf, allPluginsPath): """ Verifies that if the first stored own checkpoint has a not aligned lower bound (this means that it was started after a catch-up), is incomplete and there is a quorumed stashed checkpoint from other replicas with the same end then this stashed checkpoint is included into the lag for a catch-up, i.e. in such a case the lag which makes the node perform catch-up is Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed received checkpoints. """ max_batch_size = chkFreqPatched.Max3PCBatchSize _, new_node = sdk_add_new_steward_and_node( looper, sdk_pool_handle, sdk_wallet_steward, 'EpsilonSteward', 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Epsilon did not participate in ordering of the batch with EpsilonSteward # NYM transaction and the batch with Epsilon NODE transaction. # Epsilon got these transactions via catch-up. # Send some requests but not enough to complete the first checkpoint. # Note that there are already 2 3PC-batches in the first checkpoint: # with EpsilonSteward NYM transaction and with Epsilon NODE transaction. # This checkpoint has a not aligned lower bound on the new node replicas. send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint - 3 * max_batch_size) # The master replica of the new node stops to receive 3PC-messages new_node.master_replica.threePhaseRouter.extend( ( (PrePrepare, lambda *x, **y: None), (Prepare, lambda *x, **y: None), (Commit, lambda *x, **y: None), ) ) completed_catchups_before_reqs = get_number_of_completed_catchups(new_node) # Send requests for the new node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP - 1) * reqs_for_checkpoint + max_batch_size) # Give time for the new node to catch up if it is going to do it looper.runFor(waits.expectedPoolConsistencyProof(len(txnPoolNodeSet)) + waits.expectedPoolCatchupTime(len(txnPoolNodeSet))) checkNodeDataForInequality(new_node, *txnPoolNodeSet[:-1]) # Verify that the new node has not caught up assert get_number_of_completed_catchups(new_node) == completed_catchups_before_reqs # Send more requests for the new node's master replica to reach # Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1 quorumed stashed # checkpoints from others send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Verify that the new node has caught up assert get_number_of_completed_catchups(new_node) > completed_catchups_before_reqs