def testOrderingWhenPrePrepareNotReceived(looper, nodeSet, up, client1, wallet1): """ Send commits and prepares but delay pre-prepare such that enough prepares and commits are received, now the request should not be ordered until pre-prepare is received and ordering should just happen once, """ nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) slowRep = nonPrimReps[0] slowNode = slowRep.node slowNode.nodeIbStasher.delay(ppDelay(10, 0)) sendRandomRequest(wallet1, client1) stash = [] origMethod = slowRep.processReqDigest def patched(self, msg): stash.append(msg) patchedMethod = types.MethodType(patched, slowRep) slowRep.processReqDigest = patchedMethod def chk1(): assert len(slowRep.commitsWaitingForPrepare) > 0 looper.run(eventually(chk1, timeout=4)) for item in stash: origMethod(item) def chk2(): assert len(slowRep.commitsWaitingForPrepare) == 0 assert slowRep.spylog.count(slowRep.doOrder.__name__) == 1 looper.run(eventually(chk2, timeout=12))
def ensureAgentsConnected(looper, agent1, agent2): e1 = agent1.endpoint e2 = agent2.endpoint looper.run( eventually(checkRemoteExists, e1, e2.name, CONNECTED, timeout=10)) looper.run( eventually(checkRemoteExists, e2, e1.name, CONNECTED, timeout=10))
def sendMoney(self, to: str, amount: int, nodes, expected: bool = True): req = self.submit({ TXN_TYPE: CREDIT, TARGET_NYM: to, DATA: { AMOUNT: amount } }) if expected: self.looper.run( eventually(checkSufficientRepliesRecvd, self.client.inBox, req.reqId, 1, retryWait=1, timeout=5)) else: for node in nodes: self.looper.run( eventually(checkReqNack, self.client, node, req.identifier, req.reqId, None, retryWait=1, timeout=5)) return req
def testOrderingCase1(looper, nodeSet, up, client1, wallet1): """ Scenario -> PRE-PREPARE not received by the replica, Request not received for ordering by the replica, but received enough commits to start ordering. It queues up the request so when a PRE-PREPARE is received or request is receievd for ordering, an order can be triggered https://www.pivotaltracker.com/story/show/125239401 Reproducing by - Pick a node with no primary replica, replica ignores forwarded request to replica and delay reception of PRE-PREPARE sufficiently so that enough COMMITs reach to trigger ordering. """ replica = getNonPrimaryReplicas(nodeSet, instId=0)[0] delaysPrePrepareProcessing(replica.node, delay=10, instId=0) def doNotProcessReqDigest(self, rd: ReqDigest): pass patchedMethod = types.MethodType(doNotProcessReqDigest, replica) replica.processReqDigest = patchedMethod def chk(n): assert replica.spylog.count(replica.doOrder.__name__) == n sendRandomRequest(wallet1, client1) looper.run(eventually(chk, 0, retryWait=1, timeout=5)) looper.run(eventually(chk, 1, retryWait=1, timeout=15))
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg): """ The nodes Alpha and Beta know about each other so they should connect but they should remove remote for C when it tries to connect to them """ tdir, looper = tdirAndLooper names = ["Alpha", "Beta"] logger.debug(names) nrg = {n: nodeReg[n] for n in names} A, B = [TestNode(name, nrg, basedirpath=tdir, pluginPaths=allPluginsPath) for name in names] for node in (A, B): looper.add(node) node.startKeySharing() looper.run(checkNodesConnected([A, B])) C = TestNode("Gamma", {**nrg, **{"Gamma": nodeReg["Gamma"]}}, basedirpath=tdir, pluginPaths=allPluginsPath) looper.add(C) C.startKeySharing(timeout=20) def chk(): assert not C.nodestack.isKeySharing looper.run(eventually(chk, retryWait=2, timeout=21)) stopNodes([C, ], looper) def chk(): assert C.name not in B.nodestack.nameRemotes assert C.name not in A.nodestack.nameRemotes looper.run(eventually(chk, retryWait=2, timeout=5)) stopNodes([A, B], looper)
def testStatusAfterClientAdded(cli, validNodeNames, createAllNodes): clientName = "Joe" cli.enterCmd("new client {}".format(clientName)) cli.looper.run( eventually(checkClientConnected, cli, validNodeNames, clientName, retryWait=1, timeout=3)) cli.enterCmd("new key") cli.enterCmd("status client {}".format(clientName)) cli.looper.run( eventually(checkActiveIdrPrinted, cli, retryWait=1, timeout=3)) for name in validNodeNames: # Checking the output after command `status node <name>`. Testing # the node status here after the client is connected cli.enterCmd("status node {}".format(name)) otherNodeNames = (set(validNodeNames) - { name, }) node = cli.nodes[name] client = cli.clients[clientName] cliLogs = list(cli.printeds) if node.hasPrimary: checkPrimaryLogs(node, cliLogs) else: checkNonPrimaryLogs(node, cliLogs) checkForNamedTokens(cli.printedTokens[3], otherNodeNames) if cli.clients: checkForNamedTokens(cli.printedTokens[1], { client.stackName, })
def testPropagateRecvdBeforeRequest(setup, looper, nodeSet, up, sent1): A, B, C, D = nodeSet.nodes.values() def x(): # A should not have received a request from the client assert len(recvdRequest(A)) == 0 # A should have received only one PROPAGATE assert len(recvdPropagate(A)) == 1 # A should have sent only one PROPAGATE assert len(sentPropagate(A)) == 1 looper.run(eventually(x, retryWait=.5, timeout=3)) def y(): # A should have received a request from the client assert len(recvdRequest(A)) == 1 # A should still have sent only one PROPAGATE assert len(sentPropagate(A)) == 1 looper.run(eventually(y, retryWait=.5, timeout=6)) def chk(): # A should have forwarded the request assertLength(forwardedRequest(A), 1) looper.run(eventually(chk, retryWait=1, timeout=15))
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up): maliciousNode = nodeSet.Alpha for i in range(0, 5): maliciousNode.send(InstanceChange(i)) def chk(instId): for node in nodeSet: if node.name != maliciousNode.name: args = getAllArgs(node, Node.processInstanceChange) assert len(args) == 5 for arg in args: assert arg['frm'] == maliciousNode.name for i in range(0, 5): looper.run(eventually(chk, i, retryWait=1, timeout=20)) def g(): for node in nodeSet: if node.name != maliciousNode.name: frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == maliciousNode.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.FREQUENT_INST_CHNG.code)) == 13 looper.run(eventually(g, retryWait=1, timeout=20))
def testAnonCreds(aliceAgent, aliceAcceptedFaber, aliceAcceptedAcme, acmeAgent, emptyLooper): # 1. request Claims from Faber faberLink = aliceAgent.wallet.getLink('Faber College') name, version, origin = faberLink.availableClaims[0] claimDefKey = ClaimDefinitionKey(name, version, origin) aliceAgent.sendReqClaim(faberLink, claimDefKey) # 2. check that claim is received from Faber async def chkClaims(): claim = await aliceAgent.prover.wallet.getClaims(ID(claimDefKey)) assert claim.primaryClaim emptyLooper.run(eventually(chkClaims, timeout=20)) # 3. send claim proof to Acme acmeLink, acmeClaimPrfReq = aliceAgent.wallet.getMatchingLinksWithClaimReq( "Job-Application", "Acme Corp")[0] aliceAgent.sendProof(acmeLink, acmeClaimPrfReq) # 4. check that claim proof is verified by Acme def chkProof(): internalId = acmeAgent.getInternalIdByInvitedNonce( acmeLink.invitationNonce) link = acmeAgent.wallet.getLinkByInternalId(internalId) assert "Job-Application" in link.verifiedClaimProofs emptyLooper.run(eventually(chkProof, timeout=20))
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1): def chk(): for r in getNonPrimaryReplicas(nodeSet, instId): nodeSuspicions = len( getNodeSuspicions(r.node, Suspicions.WRONG_PPSEQ_NO.code)) assert nodeSuspicions == 1 def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares): assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \ (req.identifier, req.reqId, req.digest) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) req = propagated1.reqDigest primary.doPrePrepare(req) for np in nonPrimaryReplicas: looper.run( eventually(checkPreprepare, np, primary.viewNo, primary.lastPrePrepareSeqNo - 1, req, 1, retryWait=.5, timeout=10)) newReqDigest = ReqDigest(req.identifier, req.reqId + 1, req.digest) incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo, primary.lastPrePrepareSeqNo + 2, *newReqDigest, time.time()) primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent) looper.run(eventually(chk, retryWait=1, timeout=50))
def testAdd2NewNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): """ Add 2 new nodes to trigger replica addition and primary election """ for nodeName in ("Zeta", "Eta"): newStewardName = "testClientSteward" + randomString(3) newSteward, newStewardWallet, newNode = addNewStewardAndNode( looper, steward1, stewardWallet, newStewardName, nodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) looper.run( eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[:-1], retryWait=1, timeout=7)) f = getMaxFailures(len(txnPoolNodeSet)) def checkFValue(): for node in txnPoolNodeSet: assert node.f == f assert len(node.replicas) == (f + 1) looper.run(eventually(checkFValue, retryWait=1, timeout=5)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=5)
def nymsAddedInQuickSuccession(nodeSet, addedSponsor, looper, sponsor, sponsorWallet): usigner = SimpleSigner() nym = usigner.verkey idy = Identity(identifier=nym) sponsorWallet.addSponsoredIdentity(idy) # Creating a NYM request with same nym again req = idy.ledgerRequest() sponsorWallet._pending.appendleft((req, idy.identifier)) reqs = sponsorWallet.preparePending() sponsor.submitReqs(*reqs) def check(): assert sponsorWallet._sponsored[nym].seqNo looper.run(eventually(check, timeout=2)) looper.run( eventually(checkNacks, sponsor, req.reqId, "is already added", retryWait=1, timeout=15)) count = 0 for node in nodeSet: txns = node.domainLedger.getAllTxn() for seq, txn in txns.items(): if txn[TXN_TYPE] == NYM and txn[TARGET_NYM] == usigner.identifier: count += 1 assert (count == len(nodeSet))
def testUpgradeLatestUncancelledVersion(looper, txnPoolNodeSet, tconf, nodeThetaAdded, validUpgrade, trustee, trusteeWallet, tdirWithPoolTxns, allPluginsPath): """ A node starts and finds several upgrades but selects the latest one which is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but since 1.5 is cancelled, it selects 1.4 """ nodeSet = txnPoolNodeSet newSteward, newStewardWallet, newNode = nodeThetaAdded for node in nodeSet[:-1]: node.nodestack.removeRemoteByName(newNode.nodestack.name) newNode.nodestack.removeRemoteByName(node.nodestack.name) newNode.stop() nodeSet = nodeSet[:-1] looper.removeProdable(newNode) upgr1 = deepcopy(validUpgrade) upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(3) upgr2[SHA256] = randomString(32) upgr3 = deepcopy(upgr2) upgr3[VERSION] = bumpVersion(upgr2[VERSION]) upgr3[NAME] += randomString(3) upgr3[SHA256] = randomString(32) upgr4 = deepcopy(upgr3) upgr4[ACTION] = CANCEL ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr1[VERSION], retryWait=1, timeout=5)) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=5)) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr3) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr3[VERSION], retryWait=1, timeout=5)) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr4) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=5)) trustee.stopRetrying() newNode = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) looper.add(newNode) nodeSet.append(newNode) looper.run(checkNodesConnected(nodeSet, overrideTimeout=30)) looper.run(eventually(checkUpgradeScheduled, [newNode, ], upgr2[VERSION], retryWait=1, timeout=10))
def testReqExecWhenReturnedByMaster(tdir_for_func): with TestNodeSet(count=4, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() client1, wallet1 = setupNodesAndClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=15)) async def chk(): for node in nodeSet: entries = node.spylog.getAll(node.processOrdered.__name__) for entry in entries: arg = entry.params['ordered'] result = entry.result if arg.instId == node.instances.masterId: assert result else: assert result is None looper.run(eventually(chk, timeout=3))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The connect resends the request and gets REPLY """ alpha = nodeSet.Alpha skipped = False origTrans = alpha.transmitToClient def skipReplyOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Reply) and not skipped: skipped = True return origTrans(msg, remoteName) alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet] looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3)) looper.run(eventually(checkReplyCount, client1, *req.key, 3, retryWait=1, timeout=3)) looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1, timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def testClientRetryRequestWhenAckNotReceived(looper, nodeSet, client1, wallet1, tconf): """ The client gets disconnected from node say Alpha but does not know it. It sends request to all nodes including Alpha, expects ACK and REPLY from Alpha too, does not get it, so reconnects to Alpha and sends request again and gets REPLY """ alpha = nodeSet.Alpha r = alpha.clientstack.getRemote(client1.stackName) alpha.clientstack.removeRemote(r) req = sendRandomRequest(wallet1, client1) def chkAcks(): for node in nodeSet: if node != alpha: checkReqAck(client1, node, *req.key) else: with pytest.raises(AssertionError): checkReqAck(client1, node, *req.key) looper.run(eventually(chkAcks, retryWait=1, timeout=3)) looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1, timeout=tconf.CLIENT_REQACK_TIMEOUT+10))
def testValidatorSuspensionByTrustee(trustee, trusteeWallet, looper, nodeSet): node = nodeSet[-1] nodeNym = hexToFriendly(node.nodestack.local.signer.verhex) suspendNode(looper, trustee, trusteeWallet, nodeNym, node.name) for n in nodeSet[:-1]: looper.run(eventually(checkNodeNotInNodeReg, n, node.name)) looper.run(eventually(checkNodeNotInNodeReg, trustee, node.name))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(wallet1, client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logger.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logger.debug("Primary Replica: {}".format(primaryRepl)) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl.lastPrePrepareSeqNo -= 1 request2 = sendRandomRequest(wallet1, client1) looper.run( eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo, wallet1.defaultId, request2.reqId, request2.digest, time.time()) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run( eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run( eventually(checkPrepareReqSent, npr, wallet1.defaultId, request2.reqId, retryWait=1, timeout=10))
def changeNodeHa(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary): # prepare new ha for node and client stack subjectedNode = None stewardName = None stewardsSeed = None for nodeIndex, n in enumerate(txnPoolNodeSet): if (shouldBePrimary and n.primaryReplicaNo == 0) or \ (not shouldBePrimary and n.primaryReplicaNo != 0): subjectedNode = n stewardName = poolTxnStewardNames[nodeIndex] stewardsSeed = poolTxnData["seeds"][stewardName].encode() break nodeStackNewHA, clientStackNewHA = genHa(2) logger.debug("change HA for node: {} to {}". format(subjectedNode.name, (nodeStackNewHA, clientStackNewHA))) nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode() # change HA stewardClient, req = changeHA(looper, tconf, subjectedNode.name, nodeSeed, nodeStackNewHA, stewardName, stewardsSeed) f = getMaxFailures(len(stewardClient.nodeReg)) looper.run(eventually(checkSufficientRepliesRecvd, stewardClient.inBox, req.reqId, f, retryWait=1, timeout=20)) # stop node for which HA will be changed subjectedNode.stop() looper.removeProdable(subjectedNode) # start node with new HA restartedNode = TestNode(subjectedNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeStackNewHA, cliha=clientStackNewHA) looper.add(restartedNode) txnPoolNodeSet[nodeIndex] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=70)) ensureElectionsDone(looper, txnPoolNodeSet, retryWait=1, timeout=10) # start client and check the node HA anotherClient, _ = genTestClient(tmpdir=tdirWithPoolTxns, usePoolLedger=True) looper.add(anotherClient) looper.run(eventually(anotherClient.ensureConnectedToNodes)) stewardWallet = Wallet(stewardName) stewardWallet.addIdentifier(signer=SimpleSigner(seed=stewardsSeed)) sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, stewardClient, 8) looper.run(eventually(checkIfGenesisPoolTxnFileUpdated, *txnPoolNodeSet, stewardClient, anotherClient, retryWait=1, timeout=10))
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes): """ Primary selection (Sunny Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes A sees three other NOMINATE(A) votes (from B, C, D) A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes B sees two more NOMINATE(A) votes (from C and D) B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes C sees two more NOMINATE(A) votes (from B and D) C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes D sees two more NOMINATE(A) votes (from B and C) D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes A sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary B sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary C sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary D sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() nodesBCD = [B, C, D] checkPoolReady(looper, nodeSet) # Checking whether one of the replicas of Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) for n in nodesBCD: # Checking whether Node B, C and D nominated Node A looper.run( eventually(checkNomination, n, A.name, retryWait=1, timeout=10)) checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) assert A.hasPrimary
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, newNodeCaughtUp, tdirWithPoolTxns, tconf, allPluginsPath): """ Node discards 3-phase and election messages from view nos that it does not know of (view nos before it joined the pool) :return: """ looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns viewNo = nodeX.viewNo # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, 10) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4) looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet, viewNo + 1), retryWait=1, timeout=20)) newStewardName = "testClientSteward" + randomString(3) nodeName = "Theta" _, _, nodeTheta = addNewStewardAndNode(looper, client, wallet, newStewardName, nodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(client.ensureConnectedToNodes()) looper.run(eventually(checkNodeLedgersForEquality, nodeTheta, *txnPoolNodeSet[:-1], retryWait=1, timeout=5)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=10) electMsg = Nomination(nodeX.name, 0, viewNo) threePMsg = PrePrepare( 0, viewNo, 10, wallet.defaultId, wallet._getIdData().lastReqId+1, "random digest", time.time() ) ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid nodeX.send(electMsg, ridTheta) nodeX.send(threePMsg, ridTheta) nodeX.send(electMsg, ridTheta) looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg, 'un-acceptable viewNo', retryWait=1, timeout=5)) nodeX.send(threePMsg, ridTheta) looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg, 'un-acceptable viewNo', retryWait=1, timeout=5))
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper, txnPoolNodeSet, client1, wallet1, client1Connected): reqs = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, CHK_FREQ - 1, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1)) checkRequestCounts(txnPoolNodeSet, len(reqs)) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1)) checkRequestCounts(txnPoolNodeSet, 0) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 3 * CHK_FREQ + 1, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1)) checkRequestCounts(txnPoolNodeSet, 1)
def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, txnPoolCliNodeReg, tdirWithPoolTxns, tconf, allPluginsPath): """ Node sends catchup request to other nodes for only those sequence numbers that other nodes have. Have pool of connected nodes with some transactions made and then two more nodes say X and Y will join where Y node will start its catchup process after some time. The node starting late, i.e. Y should not receive any catchup requests :return: """ looper, _, _, _, client, wallet = nodeSetWithNodeAddedAfterSomeTxns stewardXName = "testClientStewardX" nodeXName = "Zeta" stewardYName = "testClientStewardY" nodeYName = "Eta" stewardZName = "testClientStewardZ" nodeZName = "Theta" stewardX, nodeX = addNewStewardAndNode(looper, client, stewardXName, nodeXName, tdirWithPoolTxns, tconf, allPluginsPath, autoStart=False) stewardY, nodeY = addNewStewardAndNode(looper, client, stewardYName, nodeYName, tdirWithPoolTxns, tconf, allPluginsPath, autoStart=False) nodeX.nodeIbStasher.delay(cpDelay(45)) nodeY.nodeIbStasher.delay(cpDelay(2)) looper.add(nodeX) looper.add(nodeY) txnPoolNodeSet.append(nodeX) txnPoolNodeSet.append(nodeY) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=60)) logger.debug("Stopping 2 newest nodes, {} and {}".format(nodeX.name, nodeY.name)) nodeX.stop() nodeY.stop() logger.debug("Sending requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 50) logger.debug("Starting the 2 stopped nodes, {} and {}".format(nodeX.name, nodeY.name)) nodeX.start(looper.loop) nodeY.start(looper.loop) looper.run(eventually(checkNodeLedgersForEquality, nodeX, *txnPoolNodeSet[:5], retryWait=1, timeout=15)) looper.run(eventually(checkNodeLedgersForEquality, nodeY, *txnPoolNodeSet[:5], retryWait=1, timeout=15))
def testNodeSchedulesUpgradeAfterRestart(upgradeScheduled, looper, nodeSet, validUpgrade, testNodeClass, tdirWithPoolTxns, tconf, allPluginsPath): names = [] while nodeSet: node = nodeSet.pop() names.append(node.name) node.cleanupOnStopping = False node.stop() looper.removeProdable(node) for nm in names: node = testNodeClass(nm, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(node) nodeSet.append(node) looper.run(checkNodesConnected(nodeSet)) ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) looper.run( eventually(checkUpgradeScheduled, nodeSet, validUpgrade[VERSION], retryWait=1, timeout=10))
def upgradeScheduled(validUpgradeSent, looper, nodeSet, validUpgrade): looper.run( eventually(checkUpgradeScheduled, nodeSet, validUpgrade[VERSION], retryWait=1, timeout=10))
def testNodeDetectsUpgradeDone(looper, nodeSet): def check(): for node in nodeSet: assert node.upgrader.hasCodeBeenUpgraded == \ sovrin.__metadata__.__version__ looper.run(eventually(check, retryWait=1, timeout=5))
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, wallet1, client1): """ Checking if average latency is being set """ for i in range(5): req = sendRandomRequest(wallet1, client1) looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for node in nodeSet: # type: Node mLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId, node.instances.masterId) bLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId, *node.instances.backupIds) logger.debug( "Avg. master latency : {}. Avg. backup latency: {}".format( mLat, bLat)) assert mLat > 0 assert bLat > 0
def addNewClient(role, looper, creatorClient: Client, creatorWallet: Wallet, name: str): wallet = Wallet(name) wallet.addIdentifier() idr = wallet.defaultId op = { TXN_TYPE: NYM, TARGET_NYM: idr, ALIAS: name, VERKEY: wallet.getVerkey(idr) } if role: op[ROLE] = role req = creatorWallet.signOp(op) creatorClient.submitReqs(req) nodeCount = len(creatorClient.nodeReg) looper.run( eventually(checkSufficientRepliesRecvd, creatorClient.inBox, req.reqId, 1, retryWait=1, timeout=3 * nodeCount)) return wallet
def changeNodeKeys(looper, stewardClient, stewardWallet, node, verkey): nodeNym = hexToFriendly(node.nodestack.local.signer.verhex) op = { TXN_TYPE: NODE, TARGET_NYM: nodeNym, VERKEY: verkey, DATA: { ALIAS: node.name } } req = stewardWallet.signOp(op) stewardClient.submitReqs(req) looper.run( eventually(checkSufficientRepliesRecvd, stewardClient.inBox, req.reqId, 1, retryWait=1, timeout=5)) node.nodestack.clearLocalRoleKeep() node.nodestack.clearRemoteRoleKeeps() node.nodestack.clearAllDir() node.clientstack.clearLocalRoleKeep() node.clientstack.clearRemoteRoleKeeps() node.clientstack.clearAllDir()
def changeNodeHa(looper, stewardClient, stewardWallet, node, nodeHa, clientHa): nodeNym = hexToFriendly(node.nodestack.local.signer.verhex) (nodeIp, nodePort), (clientIp, clientPort) = nodeHa, clientHa op = { TXN_TYPE: NODE, TARGET_NYM: nodeNym, DATA: { NODE_IP: nodeIp, NODE_PORT: nodePort, CLIENT_IP: clientIp, CLIENT_PORT: clientPort, ALIAS: node.name } } req = stewardWallet.signOp(op) stewardClient.submitReqs(req) looper.run( eventually(checkSufficientRepliesRecvd, stewardClient.inBox, req.reqId, 1, retryWait=1, timeout=5)) node.nodestack.clearLocalKeep() node.nodestack.clearRemoteKeeps() node.clientstack.clearLocalKeep() node.clientstack.clearRemoteKeeps()