def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): coros = [partial(checkLastClientReqForNode, node, sent1) for node in nodeSet] looper.run(eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes)) coros2 = [partial(checkReqAck, client1, node, sent1.reqId) for node in nodeSet] looper.run(eventuallyAll(*coros2, totalTimeout=5, acceptableFails=faultyNodes)) return sent1
def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): coros = [ partial(checkLastClientReqForNode, node, sent1) for node in nodeSet ] looper.run( eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes)) coros2 = [ partial(checkReqAck, client1, node, sent1.reqId) for node in nodeSet ] looper.run( eventuallyAll(*coros2, totalTimeout=5, acceptableFails=faultyNodes)) return sent1
def checkIfSameReplicaIPrimary(looper: Looper, replicas: Sequence[TestReplica] = None, retryWait: float = 1, timeout: float = 20): # One and only one primary should be found and every replica should agree # on same primary def checkElectionDone(): unknowns = sum(1 for r in replicas if r.isPrimary is None) assert unknowns == 0, "election should be complete, but {} out of {} " \ "don't know who the primary is for protocol no {}" \ .format(unknowns, len(replicas), replicas[0].instId) def checkPrisAreOne(): # number of expected primaries pris = sum(1 for r in replicas if r.isPrimary) assert pris == 1, "Primary count should be 1, but was {} for protocol no {}" \ .format(pris, replicas[0].instId) def checkPrisAreSame(): pris = {r.primaryName for r in replicas} assert len(pris) == 1, "Primary should be same for all, but were {} " \ "for protocol no {}" \ .format(pris, replicas[0].instId) looper.run( eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame, retryWait=retryWait, totalTimeout=timeout))
def checkPropagated(looper, nodeSet, request, faultyNodes=0): nodesSize = len(list(nodeSet.nodes)) # noinspection PyIncorrectDocstring def g(node: TestNode): """ 1. no of propagate received by node must be n -1 with zero faulty nodes in system; where n = num of nodes 2. no of propagate received by node must be greater than or equal to f + 1 """ actualMsgs = len([ x for x in getAllArgs(node, Node.processPropagate) if x['msg'].request[f.REQ_ID.nm] == request.reqId and x['msg'].request[f.IDENTIFIER.nm] == request.identifier and x['msg'].request[OPERATION] == request.operation ]) numOfMsgsWithZFN = nodesSize - 1 numOfMsgsWithFaults = faultyNodes + 1 assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) coros = [partial(g, node) for node in nodeSet] looper.run( eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The connect resends the request and gets REPLY """ alpha = nodeSet.Alpha skipped = False origTrans = alpha.transmitToClient def skipReplyOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Reply) and not skipped: skipped = True return origTrans(msg, remoteName) alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet] looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3)) looper.run(eventually(checkReplyCount, client1, *req.key, 3, retryWait=1, timeout=3)) looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1, timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def checkPropagated(looper, nodeSet, request, faultyNodes=0): nodesSize = len(list(nodeSet.nodes)) # noinspection PyIncorrectDocstring def g(node: TestNode): """ 1. no of propagate received by node must be n -1 with zero faulty nodes in system; where n = num of nodes 2. no of propagate received by node must be greater than or equal to f + 1 """ actualMsgs = len([x for x in getAllArgs(node, Node.processPropagate) if x['msg'].request['reqId'] == request.reqId and x['msg'].request['clientId'] == request.clientId and x['msg'].request['operation'] == request.operation]) numOfMsgsWithZFN = nodesSize - 1 numOfMsgsWithFaults = faultyNodes + 1 assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) coros = [partial(g, node) for node in nodeSet] looper.run(eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, nodeSet): update = {'op': 'REQNACK', 'reason': 'client request invalid: AssertionError amount too ' 'high\nassert 999 <= 100'} coros2 = [partial(checkReqAck, client1, node, sent1.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*coros2, totalTimeout=5))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, nodeSet): update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} coros2 = [partial(checkReqNack, client1, node, sent1.identifier, sent1.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*coros2, totalTimeout=5))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, nodeSet): update = { 'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]' } coros2 = [ partial(checkReqNack, client1, node, sent1.reqId, update) for node in nodeSet ] looper.run(eventuallyAll(*coros2, totalTimeout=5))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, nodeSet): update = { 'op': 'REQNACK', 'reason': 'client request invalid: AssertionError amount too ' 'high\nassert 999 <= 100' } coros2 = [ partial(checkReqAck, client1, node, sent1.reqId, update) for node in nodeSet ] looper.run(eventuallyAll(*coros2, totalTimeout=5))
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir, pluginVerPath): # TODO: Test more cases plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" allCoros = [] op = {TXN_TYPE: "dummy", DATA: {AMOUNT: 30}} req = submitOp(wallet1, client1, op) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid transaction " "type, must be one of {}".format(validTypes)) } allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: AUCTION_START, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: PLACE_BID, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: "some string"} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: {AMOUNT: 453}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_START, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_END, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] auctionId = str(uuid4()) op = {TXN_TYPE: PLACE_BID, DATA: {ID: auctionId, AMOUNT: -3}} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } allCoros += [ partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet ] looper.run(eventuallyAll(*allCoros, totalTimeout=5)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 0
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) validTypes = ', '.join(plugin.validTxnTypes) update = {'reason': makeReason(commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes))} coros1 = [partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros2 = [partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string"}) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros3 = [partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 }}) update = { 'reason': makeReason(commonError, "{} must be present and should be " "a number greater than 0" .format(AMOUNT))} coros4 = [partial(checkReqNack, client1, node, req.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*(coros1+coros2+coros3+coros4), totalTimeout=5)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1
def checkPrePrepared(looper, nodeSet, propagated1, instIds, faultyNodes=0): nodesSize = len(list(nodeSet)) def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([param for param in getAllArgs(primary, primary.processPrePrepare)]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare( instId, primary.viewNo, primary.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'], param['sender']) == ( expectedPrePrepareRequest, primary.name)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int(msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([param for param in getAllArgs(primary, primary.doPrePrepare) if (param['reqDigest'].clientId, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.clientId, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].clientId, param['pp'].reqId, param['pp'].digest) == ( propagated1.clientId, propagated1.reqId, propagated1.digest)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def checkCommited(looper, nodeSet, prepared1, instIds, faultyNodes=0): nodeCount = len((list(nodeSet))) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] assert rcvdCommitRqst[0] == prepared1.digest actualMsgsReceived = len(rcvdCommitRqst[1]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=60))
def checkPrePrepared(looper, nodeSet, propagated1, instIds, faultyNodes=0): nodesSize = len(list(nodeSet)) def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([ param for param in getAllArgs(primary, primary.processPrePrepare) ]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo, primary.prePrepareSeqNo, propagated1.identifier, propagated1.reqId, propagated1.digest, time.time()) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'][:-1], param['sender']) == (expectedPrePrepareRequest[:-1], primary.name) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int( msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.doPrePrepare) if (param['reqDigest'].identifier, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([ param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].identifier, param['pp'].reqId, param['pp'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def checkCommited(looper, nodeSet, prepared1, instIds, faultyNodes=0): nodeCount = len((list(nodeSet))) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] assert rcvdCommitRqst[0] == prepared1.digest actualMsgsReceived = len(rcvdCommitRqst[1]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=60))
def checkPrepared(looper, nodeSet, preprepared1, instIds, faultyNodes=0): nodeCount = len(list(nodeSet.nodes)) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.prePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) and param['sender'] != primary.name ]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) ]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir, pluginVerPath): # TODO: Test more cases plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" allCoros = [] op = { TXN_TYPE: "dummy", DATA: { AMOUNT: 30 }} req = submitOp(wallet1, client1, op) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason(commonError, "dummy is not a valid transaction " "type, must be one of {}" .format(validTypes))} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: AUCTION_START, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: PLACE_BID, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: PLACE_BID, DATA: "some string" } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: PLACE_BID, DATA: { AMOUNT: 453 }} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "No id provided for auction")} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: AUCTION_START, DATA: {} } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "No id provided for auction")} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] op = { TXN_TYPE: AUCTION_END, DATA: {} } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "No id provided for auction")} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] auctionId = str(uuid4()) op = { TXN_TYPE: PLACE_BID, DATA: { ID: auctionId, AMOUNT: -3 } } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason(commonError, "{} must be present and should be " "a number greater than 0" .format(AMOUNT))} allCoros += [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*allCoros, totalTimeout=5)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 0
def checkPrepared(looper, nodeSet, preprepared1, instIds, faultyNodes=0): nodeCount = len(list(nodeSet.nodes)) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.prePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) and param['sender'] != primary.name]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs( npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) ]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) validTypes = ', '.join(plugin.validTxnTypes) update = {'reason': makeReason(commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes))} coros1 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros2 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string"}) update = { 'reason': makeReason(commonError, "{} attribute is missing or not in proper format" .format(DATA))} coros3 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 }}) update = { 'reason': makeReason(commonError, "{} must be present and should be " "a number greater than 0" .format(AMOUNT))} coros4 = [partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet] looper.run(eventuallyAll(*(coros1+coros2+coros3+coros4), totalTimeout=5)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 }}) looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1