def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logger.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run( eventually(checkRequestReturnedToNode, node, wallet1.defaultId, req.reqId, req.digest, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logging.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run(eventually(checkRequestReturnedToNode, node, client1.defaultIdentifier, req.reqId, req.digest, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes)) } coros1 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros2 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string" }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros3 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 } }) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } coros4 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] timeout = waits.expectedReqAckQuorumTime() looper.run( eventuallyAll(*(coros1 + coros2 + coros3 + coros4), totalTimeout=timeout)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) waitForSufficientRepliesForRequests(looper, client1, requests=[req], fVal=1) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) validTypes = ', '.join(plugin.validTxnTypes) update = {'reason': makeReason(commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes))} coros1 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros2 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string"}) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros3 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 }}) update = { 'reason': makeReason(commonError, "{} must be present and should be " "a number greater than 0" .format(AMOUNT))} coros4 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*(coros1+coros2+coros3+coros4), totalTimeout=5)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1