def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, chkFreqPatched, view_change): node = txnPoolNodeSet[0] # Stabilize checkpoint # Send one more request to stabilize checkpoint sdk_send_random_and_check( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ - get_pp_seq_no(txnPoolNodeSet) % CHK_FREQ) old_stable_checkpoint = node.master_replica._consensus_data.stable_checkpoint stashers = [n.nodeIbStasher for n in txnPoolNodeSet] with delay_rules(stashers, cDelay(delay=sys.maxsize), msg_rep_delay(types_to_delay=[COMMIT])): req = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) f_d, f_r = get_forwarded_to_all(node) assert f_d node.replicas.remove_replica(node.replicas.num_replicas - 1) assert node.requests[f_d].forwardedTo == node.replicas.num_replicas check_for_nodes(txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint) sdk_get_replies(looper, req) check_for_nodes(txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint) # Send one more request to stabilize checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ - 1) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint + CHK_FREQ)) assert len(node.requests) == 0
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, chkFreqPatched, view_change): node = txnPoolNodeSet[0] stashers = [n.nodeIbStasher for n in txnPoolNodeSet] with delay_rules(stashers, cDelay(delay=sys.maxsize)): req = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) assert len(node.requests) == 1 forwardedToBefore = next(iter(node.requests.values())).forwardedTo node.replicas.remove_replica(node.replicas.num_replicas - 1) assert len(node.requests) == 1 forwardedToAfter = next(iter(node.requests.values())).forwardedTo assert forwardedToAfter == forwardedToBefore - 1 chkChkpoints(txnPoolNodeSet, 0) sdk_get_replies(looper, req) chkChkpoints(txnPoolNodeSet, 1) # Send one more request to stabilize checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0)) assert len(node.requests) == 0
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) primaryRepl = getPrimaryReplica(nodeSet, 1) logger.debug("Primary Replica: {}".format(primaryRepl)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, 1) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) # Delay COMMITs so request is not ordered and checks can be made c_delay = 10 for node in nodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) request1 = sendRandomRequest(wallet1, client1) for npr in nonPrimaryReplicas: looper.run( eventually(checkPrepareReqSent, npr, request1.identifier, request1.reqId, primaryRepl.viewNo, retryWait=1)) prePrepareReq = primaryRepl.sentPrePrepares[ primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo] looper.run( eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1)) # logger.debug("Patching the primary replica's pre-prepare sending method ") # orig_method = primaryRepl.sendPrePrepare # def patched(self, ppReq): # self.sentPrePrepares[ppReq.viewNo, ppReq.ppSeqNo] = ppReq # ppReq = updateNamedTuple(ppReq, **{f.PP_SEQ_NO.nm: 1}) # self.send(ppReq, TPCStat.PrePrepareSent) # # primaryRepl.sendPrePrepare = types.MethodType(patched, primaryRepl) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo request2 = sendRandomRequest(wallet1, client1) timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) # Since the node is malicious, it will not be able to process requests due # to conflicts in PRE-PREPARE primaryRepl.node.stop() looper.removeProdable(primaryRepl.node) reqIdr = [(request2.identifier, request2.reqId)] prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo, time.time(), reqIdr, 1, primaryRepl.batchDigest([request2]), DOMAIN_LEDGER_ID, primaryRepl.stateRootHash(DOMAIN_LEDGER_ID), primaryRepl.txnRootHash(DOMAIN_LEDGER_ID)) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=timeout)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") timeout = waits.expectedPrepareTime(len(nodeSet)) looper.runFor(timeout) # expect prepare processing timeout # check if prepares have not been sent for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run( eventually(checkPrepareReqSent, npr, request2.identifier, request2.reqId, view_no, retryWait=1, timeout=timeout)) timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) + c_delay result1 = looper.run( eventually(checkSufficientRepliesReceived, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=timeout)) logger.debug("request {} gives result {}".format(request1, result1))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay
import pytest import asyncio from plenum.test import waits from plenum.test.delayers import ppDelay, pDelay, cDelay from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_request.test_timestamp.helper import get_timestamp_suspicion_count from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.stasher import delay_rules from plenum.test.spy_helpers import get_count from stp_core.loop.eventually import eventually nodeCount = 4 # should be big enough to pass PP during normal ordering flow PATCHED_ACCEPTABLE_DEVIATION_PREPREPARE_SECS = waits.expectedPrepareTime( nodeCount) @pytest.fixture(scope="module") def tconf(tconf): old_value = tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS = PATCHED_ACCEPTABLE_DEVIATION_PREPREPARE_SECS yield tconf tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS = old_value # TODO this test should actually fail someday when ts for PP # is set before replica level processing (e.g. in zstack) def test_pp_obsolescence_check_fail_for_delayed(tdir, tconf, looper, txnPoolNodeSet, sdk_pool_handle,
def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) primaryRepl = getPrimaryReplica(txnPoolNodeSet, 1) logger.debug("Primary Replica: {}".format(primaryRepl)) nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, 1) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) # Delay COMMITs so request is not ordered and checks can be made c_delay = 10 for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) req1 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)[0] request1 = sdk_json_to_request_object(req1[0]) for npr in nonPrimaryReplicas: looper.run(eventually(checkPrepareReqSent, npr, request1.key, primaryRepl.viewNo, retryWait=1)) prePrepareReq = primaryRepl._ordering_service.sent_preprepares[primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo] looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1)) # logger.debug("Patching the primary replica's pre-prepare sending method ") # orig_method = primaryRepl.sendPrePrepare # def patched(self, ppReq): # self._ordering_service.sent_preprepares[ppReq.viewNo, ppReq.ppSeqNo] = ppReq # ppReq = updateNamedTuple(ppReq, **{f.PP_SEQ_NO.nm: 1}) # self.send(ppReq, TPCStat.PrePrepareSent) # # primaryRepl.sendPrePrepare = types.MethodType(patched, primaryRepl) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl._ordering_service._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo request2 = sdk_json_to_request_object( sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)[0][0]) timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) # Since the node is malicious, it will not be able to process requests due # to conflicts in PRE-PREPARE primaryRepl.node.stop() looper.removeProdable(primaryRepl.node) reqIdr = [request2.digest] tm = get_utc_epoch() prePrepareReq = PrePrepare( primaryRepl.instId, view_no, primaryRepl.lastPrePrepareSeqNo, tm, reqIdr, init_discarded(), primaryRepl._ordering_service.generate_pp_digest([request2.digest], view_no, tm), DOMAIN_LEDGER_ID, primaryRepl._ordering_service.get_state_root_hash(DOMAIN_LEDGER_ID), primaryRepl._ordering_service.get_txn_root_hash(DOMAIN_LEDGER_ID), 0, True ) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=timeout)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") timeout = waits.expectedPrepareTime(len(txnPoolNodeSet)) looper.runFor(timeout) # expect prepare processing timeout # check if prepares have not been sent for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, request2.key, view_no, retryWait=1, timeout=timeout)) timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay result1 = sdk_get_replies(looper, [req1])[0][1] logger.debug("request {} gives result {}".format(request1, result1))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack( nodeSet, looper, nodeNames[i]) for i in range( 0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually( lambda: assertLength( getPendingRequestsForReplica( nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay
def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) primaryRepl = getPrimaryReplica(txnPoolNodeSet, 1) logger.debug("Primary Replica: {}".format(primaryRepl)) nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, 1) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) # Delay COMMITs so request is not ordered and checks can be made c_delay = 10 for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) req1 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)[0] request1 = sdk_json_to_request_object(req1[0]) for npr in nonPrimaryReplicas: looper.run(eventually(checkPrepareReqSent, npr, request1.key, primaryRepl.viewNo, retryWait=1)) prePrepareReq = primaryRepl.sentPrePrepares[primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo] looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1)) # logger.debug("Patching the primary replica's pre-prepare sending method ") # orig_method = primaryRepl.sendPrePrepare # def patched(self, ppReq): # self.sentPrePrepares[ppReq.viewNo, ppReq.ppSeqNo] = ppReq # ppReq = updateNamedTuple(ppReq, **{f.PP_SEQ_NO.nm: 1}) # self.send(ppReq, TPCStat.PrePrepareSent) # # primaryRepl.sendPrePrepare = types.MethodType(patched, primaryRepl) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo request2 = sdk_json_to_request_object( sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)[0][0]) timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) # Since the node is malicious, it will not be able to process requests due # to conflicts in PRE-PREPARE primaryRepl.node.stop() looper.removeProdable(primaryRepl.node) reqIdr = [request2.digest] prePrepareReq = PrePrepare( primaryRepl.instId, view_no, primaryRepl.lastPrePrepareSeqNo, get_utc_epoch(), reqIdr, init_discarded(), primaryRepl.batchDigest([request2]), DOMAIN_LEDGER_ID, primaryRepl.stateRootHash(DOMAIN_LEDGER_ID), primaryRepl.txnRootHash(DOMAIN_LEDGER_ID), 0, True ) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=timeout)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") timeout = waits.expectedPrepareTime(len(txnPoolNodeSet)) looper.runFor(timeout) # expect prepare processing timeout # check if prepares have not been sent for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, request2.key, view_no, retryWait=1, timeout=timeout)) timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay result1 = sdk_get_replies(looper, [req1])[0][1] logger.debug("request {} gives result {}".format(request1, result1))