def testOrderingWhenPrePrepareNotReceived(looper, nodeSet, up, client1,
wallet1):
"""
Send commits and prepares but delay pre-prepare such that enough prepares
and commits are received, now the request should not be ordered until
pre-prepare is received and ordering should just happen once,
"""
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
slowRep = nonPrimReps[0]
slowNode = slowRep.node
slowNode.nodeIbStasher.delay(ppDelay(10, 0))
sendRandomRequest(wallet1, client1)
stash = []
origMethod = slowRep.processReqDigest
def patched(self, msg):
stash.append(msg)
patchedMethod = types.MethodType(patched, slowRep)
slowRep.processReqDigest = patchedMethod
def chk1():
assert len(slowRep.commitsWaitingForPrepare) > 0
looper.run(eventually(chk1, timeout=4))
for item in stash:
origMethod(item)
def chk2():
assert len(slowRep.commitsWaitingForPrepare) == 0
assert slowRep.spylog.count(slowRep.doOrder.__name__) == 1
looper.run(eventually(chk2, timeout=12))
def ensureAgentsConnected(looper, agent1, agent2):
e1 = agent1.endpoint
e2 = agent2.endpoint
looper.run(
eventually(checkRemoteExists, e1, e2.name, CONNECTED, timeout=10))
looper.run(
eventually(checkRemoteExists, e2, e1.name, CONNECTED, timeout=10))
def sendMoney(self, to: str, amount: int, nodes, expected: bool = True):
req = self.submit({
TXN_TYPE: CREDIT,
TARGET_NYM: to,
DATA: {
AMOUNT: amount
}
})
if expected:
self.looper.run(
eventually(checkSufficientRepliesRecvd,
self.client.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
else:
for node in nodes:
self.looper.run(
eventually(checkReqNack,
self.client,
node,
req.identifier,
req.reqId,
None,
retryWait=1,
timeout=5))
return req
def testOrderingCase1(looper, nodeSet, up, client1, wallet1):
"""
Scenario -> PRE-PREPARE not received by the replica, Request not received
for ordering by the replica, but received enough commits to start ordering.
It queues up the request so when a PRE-PREPARE is received or request is
receievd for ordering, an order can be triggered
https://www.pivotaltracker.com/story/show/125239401
Reproducing by - Pick a node with no primary replica, replica ignores
forwarded request to replica and delay reception of PRE-PREPARE sufficiently
so that enough COMMITs reach to trigger ordering.
"""
replica = getNonPrimaryReplicas(nodeSet, instId=0)[0]
delaysPrePrepareProcessing(replica.node, delay=10, instId=0)
def doNotProcessReqDigest(self, rd: ReqDigest):
pass
patchedMethod = types.MethodType(doNotProcessReqDigest, replica)
replica.processReqDigest = patchedMethod
def chk(n):
assert replica.spylog.count(replica.doOrder.__name__) == n
sendRandomRequest(wallet1, client1)
looper.run(eventually(chk, 0, retryWait=1, timeout=5))
looper.run(eventually(chk, 1, retryWait=1, timeout=15))
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
logger.debug(names)
nrg = {n: nodeReg[n] for n in names}
A, B = [TestNode(name, nrg, basedirpath=tdir,
pluginPaths=allPluginsPath)
for name in names]
for node in (A, B):
looper.add(node)
node.startKeySharing()
looper.run(checkNodesConnected([A, B]))
C = TestNode("Gamma", {**nrg, **{"Gamma": nodeReg["Gamma"]}},
basedirpath=tdir, pluginPaths=allPluginsPath)
looper.add(C)
C.startKeySharing(timeout=20)
def chk():
assert not C.nodestack.isKeySharing
looper.run(eventually(chk, retryWait=2, timeout=21))
stopNodes([C, ], looper)
def chk():
assert C.name not in B.nodestack.nameRemotes
assert C.name not in A.nodestack.nameRemotes
looper.run(eventually(chk, retryWait=2, timeout=5))
stopNodes([A, B], looper)
def testStatusAfterClientAdded(cli, validNodeNames, createAllNodes):
clientName = "Joe"
cli.enterCmd("new client {}".format(clientName))
cli.looper.run(
eventually(checkClientConnected,
cli,
validNodeNames,
clientName,
retryWait=1,
timeout=3))
cli.enterCmd("new key")
cli.enterCmd("status client {}".format(clientName))
cli.looper.run(
eventually(checkActiveIdrPrinted, cli, retryWait=1, timeout=3))
for name in validNodeNames:
# Checking the output after command `status node <name>`. Testing
# the node status here after the client is connected
cli.enterCmd("status node {}".format(name))
otherNodeNames = (set(validNodeNames) - {
name,
})
node = cli.nodes[name]
client = cli.clients[clientName]
cliLogs = list(cli.printeds)
if node.hasPrimary:
checkPrimaryLogs(node, cliLogs)
else:
checkNonPrimaryLogs(node, cliLogs)
checkForNamedTokens(cli.printedTokens[3], otherNodeNames)
if cli.clients:
checkForNamedTokens(cli.printedTokens[1], {
client.stackName,
})
def testPropagateRecvdBeforeRequest(setup, looper, nodeSet, up, sent1):
A, B, C, D = nodeSet.nodes.values()
def x():
# A should not have received a request from the client
assert len(recvdRequest(A)) == 0
# A should have received only one PROPAGATE
assert len(recvdPropagate(A)) == 1
# A should have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(x, retryWait=.5, timeout=3))
def y():
# A should have received a request from the client
assert len(recvdRequest(A)) == 1
# A should still have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(y, retryWait=.5, timeout=6))
def chk():
# A should have forwarded the request
assertLength(forwardedRequest(A), 1)
looper.run(eventually(chk, retryWait=1, timeout=15))
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up):
maliciousNode = nodeSet.Alpha
for i in range(0, 5):
maliciousNode.send(InstanceChange(i))
def chk(instId):
for node in nodeSet:
if node.name != maliciousNode.name:
args = getAllArgs(node, Node.processInstanceChange)
assert len(args) == 5
for arg in args:
assert arg['frm'] == maliciousNode.name
for i in range(0, 5):
looper.run(eventually(chk, i, retryWait=1, timeout=20))
def g():
for node in nodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node,
Suspicions.FREQUENT_INST_CHNG.code)) == 13
looper.run(eventually(g, retryWait=1, timeout=20))
def testAnonCreds(aliceAgent, aliceAcceptedFaber, aliceAcceptedAcme, acmeAgent,
emptyLooper):
# 1. request Claims from Faber
faberLink = aliceAgent.wallet.getLink('Faber College')
name, version, origin = faberLink.availableClaims[0]
claimDefKey = ClaimDefinitionKey(name, version, origin)
aliceAgent.sendReqClaim(faberLink, claimDefKey)
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaims(ID(claimDefKey))
assert claim.primaryClaim
emptyLooper.run(eventually(chkClaims, timeout=20))
# 3. send claim proof to Acme
acmeLink, acmeClaimPrfReq = aliceAgent.wallet.getMatchingLinksWithClaimReq(
"Job-Application", "Acme Corp")[0]
aliceAgent.sendProof(acmeLink, acmeClaimPrfReq)
# 4. check that claim proof is verified by Acme
def chkProof():
internalId = acmeAgent.getInternalIdByInvitedNonce(
acmeLink.invitationNonce)
link = acmeAgent.wallet.getLinkByInternalId(internalId)
assert "Job-Application" in link.verifiedClaimProofs
emptyLooper.run(eventually(chkProof, timeout=20))
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1):
def chk():
for r in getNonPrimaryReplicas(nodeSet, instId):
nodeSuspicions = len(
getNodeSuspicions(r.node, Suspicions.WRONG_PPSEQ_NO.code))
assert nodeSuspicions == 1
def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \
(req.identifier, req.reqId, req.digest)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
req = propagated1.reqDigest
primary.doPrePrepare(req)
for np in nonPrimaryReplicas:
looper.run(
eventually(checkPreprepare,
np,
primary.viewNo,
primary.lastPrePrepareSeqNo - 1,
req,
1,
retryWait=.5,
timeout=10))
newReqDigest = ReqDigest(req.identifier, req.reqId + 1, req.digest)
incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo,
primary.lastPrePrepareSeqNo + 2,
*newReqDigest, time.time())
primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent)
looper.run(eventually(chk, retryWait=1, timeout=50))
def testAdd2NewNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1,
stewardWallet, allPluginsPath):
"""
Add 2 new nodes to trigger replica addition and primary election
"""
for nodeName in ("Zeta", "Eta"):
newStewardName = "testClientSteward" + randomString(3)
newSteward, newStewardWallet, newNode = addNewStewardAndNode(
looper, steward1, stewardWallet, newStewardName, nodeName,
tdirWithPoolTxns, tconf, allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
logger.debug("{} connected to the pool".format(newNode))
looper.run(
eventually(checkNodeLedgersForEquality,
newNode,
*txnPoolNodeSet[:-1],
retryWait=1,
timeout=7))
f = getMaxFailures(len(txnPoolNodeSet))
def checkFValue():
for node in txnPoolNodeSet:
assert node.f == f
assert len(node.replicas) == (f + 1)
looper.run(eventually(checkFValue, retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=5)
def nymsAddedInQuickSuccession(nodeSet, addedSponsor, looper, sponsor,
sponsorWallet):
usigner = SimpleSigner()
nym = usigner.verkey
idy = Identity(identifier=nym)
sponsorWallet.addSponsoredIdentity(idy)
# Creating a NYM request with same nym again
req = idy.ledgerRequest()
sponsorWallet._pending.appendleft((req, idy.identifier))
reqs = sponsorWallet.preparePending()
sponsor.submitReqs(*reqs)
def check():
assert sponsorWallet._sponsored[nym].seqNo
looper.run(eventually(check, timeout=2))
looper.run(
eventually(checkNacks,
sponsor,
req.reqId,
"is already added",
retryWait=1,
timeout=15))
count = 0
for node in nodeSet:
txns = node.domainLedger.getAllTxn()
for seq, txn in txns.items():
if txn[TXN_TYPE] == NYM and txn[TARGET_NYM] == usigner.identifier:
count += 1
assert (count == len(nodeSet))
def testUpgradeLatestUncancelledVersion(looper,
txnPoolNodeSet, tconf, nodeThetaAdded,
validUpgrade, trustee, trusteeWallet,
tdirWithPoolTxns, allPluginsPath):
"""
A node starts and finds several upgrades but selects the latest one which
is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but
since 1.5 is cancelled, it selects 1.4
"""
nodeSet = txnPoolNodeSet
newSteward, newStewardWallet, newNode = nodeThetaAdded
for node in nodeSet[:-1]:
node.nodestack.removeRemoteByName(newNode.nodestack.name)
newNode.nodestack.removeRemoteByName(node.nodestack.name)
newNode.stop()
nodeSet = nodeSet[:-1]
looper.removeProdable(newNode)
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
upgr2[SHA256] = randomString(32)
upgr3 = deepcopy(upgr2)
upgr3[VERSION] = bumpVersion(upgr2[VERSION])
upgr3[NAME] += randomString(3)
upgr3[SHA256] = randomString(32)
upgr4 = deepcopy(upgr3)
upgr4[ACTION] = CANCEL
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr1[VERSION],
retryWait=1, timeout=5))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2)
looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION],
retryWait=1, timeout=5))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr3)
looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr3[VERSION],
retryWait=1, timeout=5))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr4)
looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION],
retryWait=1, timeout=5))
trustee.stopRetrying()
newNode = TestNode(newNode.name, basedirpath=tdirWithPoolTxns,
config=tconf, pluginPaths=allPluginsPath,
ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha)
looper.add(newNode)
nodeSet.append(newNode)
looper.run(checkNodesConnected(nodeSet, overrideTimeout=30))
looper.run(eventually(checkUpgradeScheduled, [newNode, ], upgr2[VERSION],
retryWait=1, timeout=10))
def testReqExecWhenReturnedByMaster(tdir_for_func):
with TestNodeSet(count=4, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
client1, wallet1 = setupNodesAndClient(looper,
nodeSet,
tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
req.reqId,
1,
retryWait=1,
timeout=15))
async def chk():
for node in nodeSet:
entries = node.spylog.getAll(node.processOrdered.__name__)
for entry in entries:
arg = entry.params['ordered']
result = entry.result
if arg.instId == node.instances.masterId:
assert result
else:
assert result is None
looper.run(eventually(chk, timeout=3))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1,
wallet1, tconf):
"""
A node say Alpha sends ACK but doesn't send REPLY. The connect resends the
request and gets REPLY
"""
alpha = nodeSet.Alpha
skipped = False
origTrans = alpha.transmitToClient
def skipReplyOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Reply) and not skipped:
skipped = True
return
origTrans(msg, remoteName)
alpha.transmitToClient = skipReplyOnce
req = sendRandomRequest(wallet1, client1)
coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet]
looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3))
looper.run(eventually(checkReplyCount, client1, *req.key, 3, retryWait=1,
timeout=3))
looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1,
timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def testClientRetryRequestWhenAckNotReceived(looper, nodeSet, client1,
wallet1, tconf):
"""
The client gets disconnected from node say Alpha but does not know it.
It sends request to all nodes including Alpha, expects ACK and REPLY from
Alpha too, does not get it, so reconnects to Alpha and sends request again
and gets REPLY
"""
alpha = nodeSet.Alpha
r = alpha.clientstack.getRemote(client1.stackName)
alpha.clientstack.removeRemote(r)
req = sendRandomRequest(wallet1, client1)
def chkAcks():
for node in nodeSet:
if node != alpha:
checkReqAck(client1, node, *req.key)
else:
with pytest.raises(AssertionError):
checkReqAck(client1, node, *req.key)
looper.run(eventually(chkAcks, retryWait=1, timeout=3))
looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1,
timeout=tconf.CLIENT_REQACK_TIMEOUT+10))
def testValidatorSuspensionByTrustee(trustee, trusteeWallet, looper, nodeSet):
node = nodeSet[-1]
nodeNym = hexToFriendly(node.nodestack.local.signer.verhex)
suspendNode(looper, trustee, trusteeWallet, nodeNym, node.name)
for n in nodeSet[:-1]:
looper.run(eventually(checkNodeNotInNodeReg, n, node.name))
looper.run(eventually(checkNodeNotInNodeReg, trustee, node.name))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(wallet1, client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
request1.reqId,
fValue,
retryWait=1,
timeout=5))
logger.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logger.debug("Primary Replica: {}".format(primaryRepl))
logger.debug(
"Decrementing the primary replica's pre-prepare sequence number by "
"one...")
primaryRepl.lastPrePrepareSeqNo -= 1
request2 = sendRandomRequest(wallet1, client1)
looper.run(
eventually(checkPrePrepareReqSent,
primaryRepl,
request2,
retryWait=1,
timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo,
primaryRepl.lastPrePrepareSeqNo,
wallet1.defaultId, request2.reqId,
request2.digest, time.time())
logger.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(
eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logger.debug("""Check that none of the non primary replicas didn't send
any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(
eventually(checkPrepareReqSent,
npr,
wallet1.defaultId,
request2.reqId,
retryWait=1,
timeout=10))
def changeNodeHa(looper, txnPoolNodeSet, tdirWithPoolTxns,
poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary):
# prepare new ha for node and client stack
subjectedNode = None
stewardName = None
stewardsSeed = None
for nodeIndex, n in enumerate(txnPoolNodeSet):
if (shouldBePrimary and n.primaryReplicaNo == 0) or \
(not shouldBePrimary and n.primaryReplicaNo != 0):
subjectedNode = n
stewardName = poolTxnStewardNames[nodeIndex]
stewardsSeed = poolTxnData["seeds"][stewardName].encode()
break
nodeStackNewHA, clientStackNewHA = genHa(2)
logger.debug("change HA for node: {} to {}".
format(subjectedNode.name, (nodeStackNewHA, clientStackNewHA)))
nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode()
# change HA
stewardClient, req = changeHA(looper, tconf, subjectedNode.name, nodeSeed,
nodeStackNewHA, stewardName, stewardsSeed)
f = getMaxFailures(len(stewardClient.nodeReg))
looper.run(eventually(checkSufficientRepliesRecvd, stewardClient.inBox,
req.reqId, f, retryWait=1, timeout=20))
# stop node for which HA will be changed
subjectedNode.stop()
looper.removeProdable(subjectedNode)
# start node with new HA
restartedNode = TestNode(subjectedNode.name, basedirpath=tdirWithPoolTxns,
config=tconf, ha=nodeStackNewHA,
cliha=clientStackNewHA)
looper.add(restartedNode)
txnPoolNodeSet[nodeIndex] = restartedNode
looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=70))
ensureElectionsDone(looper, txnPoolNodeSet, retryWait=1, timeout=10)
# start client and check the node HA
anotherClient, _ = genTestClient(tmpdir=tdirWithPoolTxns,
usePoolLedger=True)
looper.add(anotherClient)
looper.run(eventually(anotherClient.ensureConnectedToNodes))
stewardWallet = Wallet(stewardName)
stewardWallet.addIdentifier(signer=SimpleSigner(seed=stewardsSeed))
sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, stewardClient, 8)
looper.run(eventually(checkIfGenesisPoolTxnFileUpdated, *txnPoolNodeSet,
stewardClient, anotherClient, retryWait=1,
timeout=10))
def testPrimaryElectionWithAClearWinner(electContFixture, looper,
keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(
eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=10)
assert A.hasPrimary
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet,
viewNo + 1), retryWait=1, timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client,
wallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(eventually(checkNodeLedgersForEquality, nodeTheta,
*txnPoolNodeSet[:-1], retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(
0,
viewNo,
10,
wallet.defaultId,
wallet._getIdData().lastReqId+1,
"random digest",
time.time()
)
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
def testRequestOlderThanStableCheckpointRemoved(chkFreqPatched, looper,
txnPoolNodeSet, client1,
wallet1, client1Connected):
reqs = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
CHK_FREQ - 1, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1))
checkRequestCounts(txnPoolNodeSet, len(reqs))
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1))
checkRequestCounts(txnPoolNodeSet, 0)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
3 * CHK_FREQ + 1, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1))
checkRequestCounts(txnPoolNodeSet, 1)
def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns,
txnPoolCliNodeReg, tdirWithPoolTxns, tconf,
allPluginsPath):
"""
Node sends catchup request to other nodes for only those sequence numbers
that other nodes have. Have pool of connected nodes with some transactions
made and then two more nodes say X and Y will join where Y node will start
its catchup process after some time. The node starting late, i.e. Y should
not receive any catchup requests
:return:
"""
looper, _, _, _, client, wallet = nodeSetWithNodeAddedAfterSomeTxns
stewardXName = "testClientStewardX"
nodeXName = "Zeta"
stewardYName = "testClientStewardY"
nodeYName = "Eta"
stewardZName = "testClientStewardZ"
nodeZName = "Theta"
stewardX, nodeX = addNewStewardAndNode(looper, client, stewardXName,
nodeXName,
tdirWithPoolTxns, tconf,
allPluginsPath, autoStart=False)
stewardY, nodeY = addNewStewardAndNode(looper, client, stewardYName,
nodeYName,
tdirWithPoolTxns, tconf,
allPluginsPath, autoStart=False)
nodeX.nodeIbStasher.delay(cpDelay(45))
nodeY.nodeIbStasher.delay(cpDelay(2))
looper.add(nodeX)
looper.add(nodeY)
txnPoolNodeSet.append(nodeX)
txnPoolNodeSet.append(nodeY)
looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=60))
logger.debug("Stopping 2 newest nodes, {} and {}".format(nodeX.name,
nodeY.name))
nodeX.stop()
nodeY.stop()
logger.debug("Sending requests")
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 50)
logger.debug("Starting the 2 stopped nodes, {} and {}".format(nodeX.name,
nodeY.name))
nodeX.start(looper.loop)
nodeY.start(looper.loop)
looper.run(eventually(checkNodeLedgersForEquality, nodeX,
*txnPoolNodeSet[:5], retryWait=1, timeout=15))
looper.run(eventually(checkNodeLedgersForEquality, nodeY,
*txnPoolNodeSet[:5], retryWait=1, timeout=15))
def testNodeSchedulesUpgradeAfterRestart(upgradeScheduled, looper, nodeSet,
validUpgrade, testNodeClass,
tdirWithPoolTxns, tconf,
allPluginsPath):
names = []
while nodeSet:
node = nodeSet.pop()
names.append(node.name)
node.cleanupOnStopping = False
node.stop()
looper.removeProdable(node)
for nm in names:
node = testNodeClass(nm,
basedirpath=tdirWithPoolTxns,
config=tconf,
pluginPaths=allPluginsPath)
looper.add(node)
nodeSet.append(node)
looper.run(checkNodesConnected(nodeSet))
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=10)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
validUpgrade[VERSION],
retryWait=1,
timeout=10))
def upgradeScheduled(validUpgradeSent, looper, nodeSet, validUpgrade):
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
validUpgrade[VERSION],
retryWait=1,
timeout=10))
def testNodeDetectsUpgradeDone(looper, nodeSet):
def check():
for node in nodeSet:
assert node.upgrader.hasCodeBeenUpgraded == \
sovrin.__metadata__.__version__
looper.run(eventually(check, retryWait=1, timeout=5))
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, wallet1, client1):
"""
Checking if average latency is being set
"""
for i in range(5):
req = sendRandomRequest(wallet1, client1)
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
for node in nodeSet: # type: Node
mLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId,
node.instances.masterId)
bLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId,
*node.instances.backupIds)
logger.debug(
"Avg. master latency : {}. Avg. backup latency: {}".format(
mLat, bLat))
assert mLat > 0
assert bLat > 0
def addNewClient(role, looper, creatorClient: Client, creatorWallet: Wallet,
name: str):
wallet = Wallet(name)
wallet.addIdentifier()
idr = wallet.defaultId
op = {
TXN_TYPE: NYM,
TARGET_NYM: idr,
ALIAS: name,
VERKEY: wallet.getVerkey(idr)
}
if role:
op[ROLE] = role
req = creatorWallet.signOp(op)
creatorClient.submitReqs(req)
nodeCount = len(creatorClient.nodeReg)
looper.run(
eventually(checkSufficientRepliesRecvd,
creatorClient.inBox,
req.reqId,
1,
retryWait=1,
timeout=3 * nodeCount))
return wallet
def changeNodeKeys(looper, stewardClient, stewardWallet, node, verkey):
nodeNym = hexToFriendly(node.nodestack.local.signer.verhex)
op = {
TXN_TYPE: NODE,
TARGET_NYM: nodeNym,
VERKEY: verkey,
DATA: {
ALIAS: node.name
}
}
req = stewardWallet.signOp(op)
stewardClient.submitReqs(req)
looper.run(
eventually(checkSufficientRepliesRecvd,
stewardClient.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
node.nodestack.clearLocalRoleKeep()
node.nodestack.clearRemoteRoleKeeps()
node.nodestack.clearAllDir()
node.clientstack.clearLocalRoleKeep()
node.clientstack.clearRemoteRoleKeeps()
node.clientstack.clearAllDir()
def changeNodeHa(looper, stewardClient, stewardWallet, node, nodeHa, clientHa):
nodeNym = hexToFriendly(node.nodestack.local.signer.verhex)
(nodeIp, nodePort), (clientIp, clientPort) = nodeHa, clientHa
op = {
TXN_TYPE: NODE,
TARGET_NYM: nodeNym,
DATA: {
NODE_IP: nodeIp,
NODE_PORT: nodePort,
CLIENT_IP: clientIp,
CLIENT_PORT: clientPort,
ALIAS: node.name
}
}
req = stewardWallet.signOp(op)
stewardClient.submitReqs(req)
looper.run(
eventually(checkSufficientRepliesRecvd,
stewardClient.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
node.nodestack.clearLocalKeep()
node.nodestack.clearRemoteKeeps()
node.clientstack.clearLocalKeep()
node.clientstack.clearRemoteKeeps()
