def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes):
numerOfNodes = len(nodeSet)
# Wait until request received by all nodes
propTimeout = waits.expectedClientToPoolRequestDeliveryTime(numerOfNodes)
coros = [
partial(checkLastClientReqForNode, node, sent1) for node in nodeSet
]
# looper.run(eventuallyAll(*coros,
# totalTimeout=propTimeout,
# acceptableFails=faultyNodes))
chk_all_funcs(looper,
coros,
acceptable_fails=faultyNodes,
timeout=propTimeout)
# Wait until sufficient number of acks received
coros2 = [
partial(checkReqAck, client1, node, sent1.identifier, sent1.reqId)
for node in nodeSet
]
ackTimeout = waits.expectedReqAckQuorumTime()
# looper.run(eventuallyAll(*coros2,
# totalTimeout=ackTimeout,
# acceptableFails=faultyNodes))
chk_all_funcs(looper,
coros2,
acceptable_fails=faultyNodes,
timeout=ackTimeout)
return sent1
def check_reqacks(client, looper, reqs, txnPoolNodeSet):
reqack_coros = []
for req in reqs:
reqack_coros.extend([
partial(checkReqAck, client, node, req.identifier, req.reqId, None)
for node in txnPoolNodeSet
])
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*reqack_coros, totalTimeout=timeout))
def ensureUpgradeSent(looper, trustee, trusteeWallet, upgradeData):
upgrade, req = sendUpgrade(trustee, trusteeWallet, upgradeData)
waitForSufficientRepliesForRequests(looper, trustee, requests=[req])
def check():
assert trusteeWallet.getPoolUpgrade(upgrade.key).seqNo
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(check, retryWait=1, timeout=timeout))
return upgrade
def ensurePoolConfigSent(looper, trustee, trusteeWallet, sendPoolCfg):
poolCfg, req = sendPoolConfig(trustee, trusteeWallet, sendPoolCfg)
waitForSufficientRepliesForRequests(looper, trustee, requests=[req])
def check():
assert trusteeWallet.getPoolConfig(poolCfg.key).seqNo
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(check, retryWait=1, timeout=timeout))
return poolCfg
def testRetrieveEmptyVerkey(didAddedWithoutVerkey, looper, trustAnchor,
trustAnchorWallet, noKeyIdr):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=noKeyIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
assert trustAnchorWallet.getIdentity(noKeyIdr).verkey is None
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testRetrieveEmptyVerkey(didAddedWithoutVerkey, looper, trustAnchor,
trustAnchorWallet, noKeyIdr):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=noKeyIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
assert trustAnchorWallet.getIdentity(noKeyIdr).verkey is None
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def updateIndyIdrWithVerkey(looper, senderWallet, senderClient, idr, fullKey):
idy = Identity(identifier=idr, verkey=fullKey)
senderWallet.updateTrustAnchoredIdentity(idy)
# TODO: What if the request fails, there must be some rollback mechanism
assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is None
reqs = senderWallet.preparePending()
senderClient.submitReqs(*reqs)
def chk():
assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is not None
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def updateSovrinIdrWithVerkey(looper, senderWallet, senderClient, idr, fullKey):
idy = Identity(identifier=idr, verkey=fullKey)
senderWallet.updateTrustAnchoredIdentity(idy)
# TODO: What if the request fails, there must be some rollback mechanism
assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is None
reqs = senderWallet.preparePending()
senderClient.submitReqs(*reqs)
def chk():
assert senderWallet.getTrustAnchoredIdentity(idr).seqNo is not None
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def fetchFullVerkeyFromIndy(looper, senderWallet, senderClient,
ownerWallet, idr):
identity = Identity(identifier=idr)
req = senderWallet.requestIdentity(identity, sender=senderWallet.defaultId)
senderClient.submitReqs(req)
def chk():
retrievedVerkey = senderWallet.getIdentity(idr).verkey
assertEquality(retrievedVerkey, ownerWallet.getVerkey(idr))
checkFullVerkeySize(retrievedVerkey)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def fetchFullVerkeyFromSovrin(looper, senderWallet, senderClient, ownerWallet,
idr):
identity = Identity(identifier=idr)
req = senderWallet.requestIdentity(identity, sender=senderWallet.defaultId)
senderClient.submitReqs(req)
def chk():
retrievedVerkey = senderWallet.getIdentity(idr).verkey
assertEquality(retrievedVerkey, ownerWallet.getVerkey(idr))
checkFullVerkeySize(retrievedVerkey)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testRequestFullRoundTrip(restrictiveVerifier,
client1,
sent1,
looper,
txnPoolNodeSet):
update = {'reason': 'client request invalid: InvalidClientRequest() '
'[caused by amount too high\nassert 999 <= 100]'}
coros2 = [partial(checkReqNack, client1, node, sent1.identifier,
sent1.reqId, update)
for node in txnPoolNodeSet]
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*coros2, totalTimeout=timeout))
def verkeyFetched(didUpdatedWithVerkey, looper, trustAnchor, trustAnchorWallet,
noKeyIdr, wallet):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=noKeyIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
assert trustAnchorWallet.getIdentity(
noKeyIdr).verkey == wallet.getVerkey(noKeyIdr)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def verkeyFetched(didUpdatedWithVerkey, looper, trustAnchor, trustAnchorWallet,
noKeyIdr, wallet):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=noKeyIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
assert trustAnchorWallet.getIdentity(
noKeyIdr).verkey == wallet.getVerkey(noKeyIdr)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def checkSubmitted(looper, client, optype, txnsBefore):
txnsAfter = []
def checkTxnCountAdvanced():
nonlocal txnsAfter
txnsAfter = client.getTxnsByType(optype)
logger.debug("old and new txns {} {}".format(txnsBefore, txnsAfter))
assert len(txnsAfter) > len(txnsBefore)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(checkTxnCountAdvanced, retryWait=1, timeout=timeout))
txnIdsBefore = [get_txn_id(txn) for txn in txnsBefore]
txnIdsAfter = [get_txn_id(txn) for txn in txnsAfter]
logger.debug("old and new txnids {} {}".format(txnIdsBefore, txnIdsAfter))
return list(set(txnIdsAfter) - set(txnIdsBefore))
def testRetrieveAbbrvVerkey(didAddedWithAbbrvVerkey, looper, trustAnchor,
trustAnchorWallet, wallet, abbrevIdr):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=abbrevIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
retrievedVerkey = trustAnchorWallet.getIdentity(abbrevIdr).verkey
assertEquality(retrievedVerkey, wallet.getVerkey(abbrevIdr))
checkAbbrVerkeySize(retrievedVerkey)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
chkVerifyForRetrievedIdentity(wallet, trustAnchorWallet, abbrevIdr)
def testRetrieveFullVerkey(didAddedWithFullVerkey, looper, trustAnchor,
trustAnchorWallet, wallet, fullKeyIdr):
"""{ type: GET_NYM, dest: <id1> }"""
identity = Identity(identifier=fullKeyIdr)
req = trustAnchorWallet.requestIdentity(identity,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
def chk():
retrievedVerkey = trustAnchorWallet.getIdentity(fullKeyIdr).verkey
assert retrievedVerkey == wallet.getVerkey(fullKeyIdr)
checkFullVerkeySize(retrievedVerkey)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(chk, retryWait=1, timeout=timeout))
chkVerifyForRetrievedIdentity(wallet, trustAnchorWallet, fullKeyIdr)
def checkSubmitted(looper, client, optype, txnsBefore):
txnsAfter = []
def checkTxnCountAdvanced():
nonlocal txnsAfter
txnsAfter = client.getTxnsByType(optype)
logger.debug("old and new txns {} {}".format(txnsBefore, txnsAfter))
assert len(txnsAfter) > len(txnsBefore)
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(checkTxnCountAdvanced, retryWait=1,
timeout=timeout))
txnIdsBefore = [txn[TXN_ID] for txn in txnsBefore]
txnIdsAfter = [txn[TXN_ID] for txn in txnsAfter]
logger.debug("old and new txnids {} {}".format(txnIdsBefore, txnIdsAfter))
return list(set(txnIdsAfter) - set(txnIdsBefore))
def testClientNotRetryRequestWhenReqnackReceived(looper, nodeSet, client1,
wallet1):
"""
A node sends REQNACK. The client does not resend Request.
"""
numOfNodes = len(nodeSet)
alpha = nodeSet.Alpha
origProcReq = alpha.processRequest
origTrans = alpha.transmitToClient
def nackReq(self, req, frm):
self.transmitToClient(RequestNack(*req.key, "testing"), frm)
def onlyTransNack(msg, remoteName):
if not isinstance(msg, RequestNack):
return
origTrans(msg, remoteName)
alpha.clientMsgRouter.routes[Request] = types.MethodType(nackReq, alpha)
alpha.transmitToClient = onlyTransNack
totalResends = client1.spylog.count(client1.resendRequests.__name__)
req = sendRandomRequest(wallet1, client1)
reqAckTimeout = waits.expectedReqAckQuorumTime()
executionTimeout = waits.expectedTransactionExecutionTime(numOfNodes)
# Wait till ACK timeout
looper.runFor(reqAckTimeout + 1)
assert client1.spylog.count(
client1.resendRequests.__name__) == totalResends
# Wait till REPLY timeout
retryTimeout = executionTimeout - reqAckTimeout + 1
looper.runFor(retryTimeout)
assert client1.spylog.count(
client1.resendRequests.__name__) == totalResends
idr, reqId = req.key
wait_for_replies(looper, client1, idr, reqId, 3)
alpha.clientMsgRouter.routes[Request] = origProcReq
alpha.transmitToClient = origTrans
async def checkAcks(self, org=None, count=1, minusInBox=None):
org = org if org else self.actor
ib = self.copyOfInBox(org)
if minusInBox:
for x in minusInBox:
ib.remove(x)
timeout = plenumWaits.expectedReqAckQuorumTime()
for node in self.nodes:
await eventually(self.checkInboxForReAck,
org.client.name,
ib,
REQACK,
node,
count,
retryWait=.1,
timeout=timeout,
ratchetSteps=10)
async def checkAcks(self, org=None, count=1, minusInBox=None):
org = org if org else self.actor
ib = self.copyOfInBox(org)
if minusInBox:
for x in minusInBox:
ib.remove(x)
timeout = plenumWaits.expectedReqAckQuorumTime()
for node in self.nodes:
await eventually(self.checkInboxForReAck,
org.client.name,
ib,
REQACK,
node,
count,
retryWait=.1,
timeout=timeout,
ratchetSteps=10)
def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1,
wallet1, tconf):
"""
A node say Alpha sends ACK but doesn't send REPLY. The client resends the
request and gets REPLY
"""
alpha = txnPoolNodeSet[0]
skipped = False
origTrans = alpha.transmitToClient
def skipReplyOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Reply) and not skipped:
skipped = True
return
origTrans(msg, remoteName)
alpha.transmitToClient = skipReplyOnce
req = sendRandomRequest(wallet1, client1)
coros = [
partial(checkReqAck, client1, node, *req.key)
for node in txnPoolNodeSet
]
timeout = waits.expectedReqAckQuorumTime()
start = time.perf_counter()
looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout))
idr, reqId = req.key
# Client should get only 3 replies till the retry timeout since one node
# is not sending any replies
wait_for_replies(looper,
client1,
idr,
reqId,
3,
custom_timeout=tconf.CLIENT_REPLY_TIMEOUT - 1)
end = time.perf_counter()
# Client should wait till the retry timeout but after that should
# get the reply from the remaining node
looper.runFor(tconf.CLIENT_REPLY_TIMEOUT - (end - start))
wait_for_replies(looper, client1, idr, reqId, 4)
def testClientRetryRequestWhenAckNotReceived(looper, nodeSet, client1,
wallet1):
"""
The client gets disconnected from node say Alpha but does not know it.
It sends request to all nodes including Alpha, expects ACK and REPLY from
Alpha too, does not get it, so reconnects to Alpha and sends request again
and gets REPLY
"""
alpha = nodeSet.Alpha
skipped = False
origPr = alpha.processRequest
def skipReqOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Request) and not skipped:
skipped = True
return
origPr(msg, remoteName)
alpha.clientMsgRouter.routes[Request] = skipReqOnce
req = sendRandomRequest(wallet1, client1)
def chkAcks():
for node in nodeSet:
if node != alpha:
checkReqAck(client1, node, *req.key)
else:
with pytest.raises(AssertionError):
checkReqAck(client1, node, *req.key)
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventually(chkAcks, retryWait=1, timeout=timeout))
idr, reqId = req.key
wait_for_replies(looper, client1, idr, reqId, 4)
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir,
pluginVerPath):
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir)
req = submitOp(wallet1, client1, {
TXN_TYPE: "dummy",
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid "
"transaction type, must be "
"one of {}".format(validTypes))
}
coros1 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros2 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: "some string"
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros3 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: -3
}
})
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
coros4 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
timeout = waits.expectedReqAckQuorumTime()
looper.run(
eventuallyAll(*(coros1 + coros2 + coros3 + coros4),
totalTimeout=timeout))
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
waitForSufficientRepliesForRequests(looper,
client1,
requests=[req],
fVal=1)
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 1
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir,
pluginVerPath):
# TODO: Test more cases
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
allCoros = []
op = {TXN_TYPE: "dummy", DATA: {AMOUNT: 30}}
req = submitOp(wallet1, client1, op)
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid transaction "
"type, must be one of {}".format(validTypes))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {
TXN_TYPE: AUCTION_START,
}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {
TXN_TYPE: PLACE_BID,
}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: PLACE_BID, DATA: "some string"}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: PLACE_BID, DATA: {AMOUNT: 453}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: AUCTION_START, DATA: {}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: AUCTION_END, DATA: {}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
auctionId = str(uuid4())
op = {TXN_TYPE: PLACE_BID, DATA: {ID: auctionId, AMOUNT: -3}}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*allCoros, totalTimeout=timeout))
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 0
