def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): numerOfNodes = len(nodeSet) # Wait until request received by all nodes propTimeout = waits.expectedClientToPoolRequestDeliveryTime(numerOfNodes) coros = [ partial(checkLastClientReqForNode, node, sent1) for node in nodeSet ] # looper.run(eventuallyAll(*coros, # totalTimeout=propTimeout, # acceptableFails=faultyNodes)) chk_all_funcs(looper, coros, acceptable_fails=faultyNodes, timeout=propTimeout) # Wait until sufficient number of acks received coros2 = [ partial(checkReqAck, client1, node, sent1.identifier, sent1.reqId) for node in nodeSet ] ackTimeout = waits.expectedReqAckQuorumTime() # looper.run(eventuallyAll(*coros2, # totalTimeout=ackTimeout, # acceptableFails=faultyNodes)) chk_all_funcs(looper, coros2, acceptable_fails=faultyNodes, timeout=ackTimeout) return sent1
def check_reqacks(client, looper, reqs, txnPoolNodeSet): reqack_coros = [] for req in reqs: reqack_coros.extend([ partial(checkReqAck, client, node, req.identifier, req.reqId, None) for node in txnPoolNodeSet ]) timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*reqack_coros, totalTimeout=timeout))
def ensureUpgradeSent(looper, trustee, trusteeWallet, upgradeData): upgrade, req = sendUpgrade(trustee, trusteeWallet, upgradeData) waitForSufficientRepliesForRequests(looper, trustee, requests=[req]) def check(): assert trusteeWallet.getPoolUpgrade(upgrade.key).seqNo timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(check, retryWait=1, timeout=timeout)) return upgrade
def ensurePoolConfigSent(looper, trustee, trusteeWallet, sendPoolCfg): poolCfg, req = sendPoolConfig(trustee, trusteeWallet, sendPoolCfg) waitForSufficientRepliesForRequests(looper, trustee, requests=[req]) def check(): assert trusteeWallet.getPoolConfig(poolCfg.key).seqNo timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(check, retryWait=1, timeout=timeout)) return poolCfg
def testRetrieveEmptyVerkey(didAddedWithoutVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=noKeyIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): assert trustAnchorWallet.getIdentity(noKeyIdr).verkey is None timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testRetrieveEmptyVerkey(didAddedWithoutVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=noKeyIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): assert trustAnchorWallet.getIdentity(noKeyIdr).verkey is None timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def updateIndyIdrWithVerkey(looper, senderWallet, senderClient, idr, fullKey): idy = Identity(identifier=idr, verkey=fullKey) senderWallet.updateTrustAnchoredIdentity(idy) # TODO: What if the request fails, there must be some rollback mechanism assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is None reqs = senderWallet.preparePending() senderClient.submitReqs(*reqs) def chk(): assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is not None timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def updateSovrinIdrWithVerkey(looper, senderWallet, senderClient, idr, fullKey): idy = Identity(identifier=idr, verkey=fullKey) senderWallet.updateTrustAnchoredIdentity(idy) # TODO: What if the request fails, there must be some rollback mechanism assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is None reqs = senderWallet.preparePending() senderClient.submitReqs(*reqs) def chk(): assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is not None timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def fetchFullVerkeyFromIndy(looper, senderWallet, senderClient, ownerWallet, idr): identity = Identity(identifier=idr) req = senderWallet.requestIdentity(identity, sender=senderWallet.defaultId) senderClient.submitReqs(req) def chk(): retrievedVerkey = senderWallet.getIdentity(idr).verkey assertEquality(retrievedVerkey, ownerWallet.getVerkey(idr)) checkFullVerkeySize(retrievedVerkey) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def fetchFullVerkeyFromSovrin(looper, senderWallet, senderClient, ownerWallet, idr): identity = Identity(identifier=idr) req = senderWallet.requestIdentity(identity, sender=senderWallet.defaultId) senderClient.submitReqs(req) def chk(): retrievedVerkey = senderWallet.getIdentity(idr).verkey assertEquality(retrievedVerkey, ownerWallet.getVerkey(idr)) checkFullVerkeySize(retrievedVerkey) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, txnPoolNodeSet): update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} coros2 = [partial(checkReqNack, client1, node, sent1.identifier, sent1.reqId, update) for node in txnPoolNodeSet] timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*coros2, totalTimeout=timeout))
def verkeyFetched(didUpdatedWithVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr, wallet): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=noKeyIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): assert trustAnchorWallet.getIdentity( noKeyIdr).verkey == wallet.getVerkey(noKeyIdr) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def verkeyFetched(didUpdatedWithVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr, wallet): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=noKeyIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): assert trustAnchorWallet.getIdentity( noKeyIdr).verkey == wallet.getVerkey(noKeyIdr) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout))
def checkSubmitted(looper, client, optype, txnsBefore): txnsAfter = [] def checkTxnCountAdvanced(): nonlocal txnsAfter txnsAfter = client.getTxnsByType(optype) logger.debug("old and new txns {} {}".format(txnsBefore, txnsAfter)) assert len(txnsAfter) > len(txnsBefore) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(checkTxnCountAdvanced, retryWait=1, timeout=timeout)) txnIdsBefore = [get_txn_id(txn) for txn in txnsBefore] txnIdsAfter = [get_txn_id(txn) for txn in txnsAfter] logger.debug("old and new txnids {} {}".format(txnIdsBefore, txnIdsAfter)) return list(set(txnIdsAfter) - set(txnIdsBefore))
def testRetrieveAbbrvVerkey(didAddedWithAbbrvVerkey, looper, trustAnchor, trustAnchorWallet, wallet, abbrevIdr): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=abbrevIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): retrievedVerkey = trustAnchorWallet.getIdentity(abbrevIdr).verkey assertEquality(retrievedVerkey, wallet.getVerkey(abbrevIdr)) checkAbbrVerkeySize(retrievedVerkey) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout)) chkVerifyForRetrievedIdentity(wallet, trustAnchorWallet, abbrevIdr)
def testRetrieveFullVerkey(didAddedWithFullVerkey, looper, trustAnchor, trustAnchorWallet, wallet, fullKeyIdr): """{ type: GET_NYM, dest: <id1> }""" identity = Identity(identifier=fullKeyIdr) req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): retrievedVerkey = trustAnchorWallet.getIdentity(fullKeyIdr).verkey assert retrievedVerkey == wallet.getVerkey(fullKeyIdr) checkFullVerkeySize(retrievedVerkey) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout)) chkVerifyForRetrievedIdentity(wallet, trustAnchorWallet, fullKeyIdr)
def checkSubmitted(looper, client, optype, txnsBefore): txnsAfter = [] def checkTxnCountAdvanced(): nonlocal txnsAfter txnsAfter = client.getTxnsByType(optype) logger.debug("old and new txns {} {}".format(txnsBefore, txnsAfter)) assert len(txnsAfter) > len(txnsBefore) timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(checkTxnCountAdvanced, retryWait=1, timeout=timeout)) txnIdsBefore = [txn[TXN_ID] for txn in txnsBefore] txnIdsAfter = [txn[TXN_ID] for txn in txnsAfter] logger.debug("old and new txnids {} {}".format(txnIdsBefore, txnIdsAfter)) return list(set(txnIdsAfter) - set(txnIdsBefore))
def testClientNotRetryRequestWhenReqnackReceived(looper, nodeSet, client1, wallet1): """ A node sends REQNACK. The client does not resend Request. """ numOfNodes = len(nodeSet) alpha = nodeSet.Alpha origProcReq = alpha.processRequest origTrans = alpha.transmitToClient def nackReq(self, req, frm): self.transmitToClient(RequestNack(*req.key, "testing"), frm) def onlyTransNack(msg, remoteName): if not isinstance(msg, RequestNack): return origTrans(msg, remoteName) alpha.clientMsgRouter.routes[Request] = types.MethodType(nackReq, alpha) alpha.transmitToClient = onlyTransNack totalResends = client1.spylog.count(client1.resendRequests.__name__) req = sendRandomRequest(wallet1, client1) reqAckTimeout = waits.expectedReqAckQuorumTime() executionTimeout = waits.expectedTransactionExecutionTime(numOfNodes) # Wait till ACK timeout looper.runFor(reqAckTimeout + 1) assert client1.spylog.count( client1.resendRequests.__name__) == totalResends # Wait till REPLY timeout retryTimeout = executionTimeout - reqAckTimeout + 1 looper.runFor(retryTimeout) assert client1.spylog.count( client1.resendRequests.__name__) == totalResends idr, reqId = req.key wait_for_replies(looper, client1, idr, reqId, 3) alpha.clientMsgRouter.routes[Request] = origProcReq alpha.transmitToClient = origTrans
async def checkAcks(self, org=None, count=1, minusInBox=None): org = org if org else self.actor ib = self.copyOfInBox(org) if minusInBox: for x in minusInBox: ib.remove(x) timeout = plenumWaits.expectedReqAckQuorumTime() for node in self.nodes: await eventually(self.checkInboxForReAck, org.client.name, ib, REQACK, node, count, retryWait=.1, timeout=timeout, ratchetSteps=10)
async def checkAcks(self, org=None, count=1, minusInBox=None): org = org if org else self.actor ib = self.copyOfInBox(org) if minusInBox: for x in minusInBox: ib.remove(x) timeout = plenumWaits.expectedReqAckQuorumTime() for node in self.nodes: await eventually(self.checkInboxForReAck, org.client.name, ib, REQACK, node, count, retryWait=.1, timeout=timeout, ratchetSteps=10)
def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The client resends the request and gets REPLY """ alpha = txnPoolNodeSet[0] skipped = False origTrans = alpha.transmitToClient def skipReplyOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Reply) and not skipped: skipped = True return origTrans(msg, remoteName) alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) coros = [ partial(checkReqAck, client1, node, *req.key) for node in txnPoolNodeSet ] timeout = waits.expectedReqAckQuorumTime() start = time.perf_counter() looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout)) idr, reqId = req.key # Client should get only 3 replies till the retry timeout since one node # is not sending any replies wait_for_replies(looper, client1, idr, reqId, 3, custom_timeout=tconf.CLIENT_REPLY_TIMEOUT - 1) end = time.perf_counter() # Client should wait till the retry timeout but after that should # get the reply from the remaining node looper.runFor(tconf.CLIENT_REPLY_TIMEOUT - (end - start)) wait_for_replies(looper, client1, idr, reqId, 4)
def testClientRetryRequestWhenAckNotReceived(looper, nodeSet, client1, wallet1): """ The client gets disconnected from node say Alpha but does not know it. It sends request to all nodes including Alpha, expects ACK and REPLY from Alpha too, does not get it, so reconnects to Alpha and sends request again and gets REPLY """ alpha = nodeSet.Alpha skipped = False origPr = alpha.processRequest def skipReqOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Request) and not skipped: skipped = True return origPr(msg, remoteName) alpha.clientMsgRouter.routes[Request] = skipReqOnce req = sendRandomRequest(wallet1, client1) def chkAcks(): for node in nodeSet: if node != alpha: checkReqAck(client1, node, *req.key) else: with pytest.raises(AssertionError): checkReqAck(client1, node, *req.key) timeout = waits.expectedReqAckQuorumTime() looper.run(eventually(chkAcks, retryWait=1, timeout=timeout)) idr, reqId = req.key wait_for_replies(looper, client1, idr, reqId, 4)
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes)) } coros1 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros2 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string" }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros3 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 } }) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } coros4 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] timeout = waits.expectedReqAckQuorumTime() looper.run( eventuallyAll(*(coros1 + coros2 + coros3 + coros4), totalTimeout=timeout)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) waitForSufficientRepliesForRequests(looper, client1, requests=[req], fVal=1) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir, pluginVerPath): # TODO: Test more cases plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" allCoros = [] op = {TXN_TYPE: "dummy", DATA: {AMOUNT: 30}} req = submitOp(wallet1, client1, op) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid transaction " "type, must be one of {}".format(validTypes)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: AUCTION_START, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: PLACE_BID, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: "some string"} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: {AMOUNT: 453}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_START, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_END, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] auctionId = str(uuid4()) op = {TXN_TYPE: PLACE_BID, DATA: {ID: auctionId, AMOUNT: -3}} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*allCoros, totalTimeout=timeout)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 0