def checkProtocolInstanceSetup(looper: Looper, nodes: Sequence[TestNode], retryWait: float = 1, customTimeout: float = None, instances: Sequence[int] = None, check_primaries=True): timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes)) checkEveryProtocolInstanceHasOnlyOnePrimary(looper=looper, nodes=nodes, retryWait=retryWait, timeout=timeout, instances_list=instances) checkEveryNodeHasAtMostOnePrimary(looper=looper, nodes=nodes, retryWait=retryWait, customTimeout=timeout) def check_not_in_view_change(): assert all([not n.master_replica._consensus_data.waiting_for_new_view for n in nodes]) looper.run(eventually(check_not_in_view_change, retryWait=retryWait, timeout=customTimeout)) if check_primaries: for n in nodes[1:]: assert nodes[0].primaries == n.primaries primaryReplicas = {replica.instId: replica for node in nodes for replica in node.replicas.values() if replica.isPrimary} return [r[1] for r in sorted(primaryReplicas.items(), key=operator.itemgetter(0))]
def checkIfSameReplicaIsPrimary(looper: Looper, replicas: Sequence[TestReplica] = None, retryWait: float = 1, timeout: float = 20): # One and only one primary should be found and every replica should agree # on same primary def checkElectionDone(): unknowns = [r for r in replicas if r.primaryName is None] assert len(unknowns) == 0, "election should be complete, " \ "but {} out of {} ({}) don't know who the primary " \ "is for protocol instance {}". \ format(len(unknowns), len(replicas), unknowns, replicas[0].instId) def checkPrisAreOne(): # number of expected primaries pris = sum(1 for r in replicas if r.isPrimary) assert pris == 1, "Primary count should be 1, but was {} for " \ "protocol no {}".format(pris, replicas[0].instId) def checkPrisAreSame(): pris = {r.primaryName for r in replicas} assert len(pris) == 1, "Primary should be same for all, but were {} " \ "for protocol no {}" \ .format(pris, replicas[0].instId) looper.run( eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame, retryWait=retryWait, totalTimeout=timeout))
def setupNodesAndClient(looper: Looper, nodes: Sequence[TestNode], nodeReg=None, tmpdir=None): looper.run(checkNodesConnected(nodes)) ensureElectionsDone(looper=looper, nodes=nodes) return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def ensure_node_disconnected(looper: Looper, disconnected: TestNode, other_nodes: Iterable[TestNode], timeout: float = None): timeout = timeout or (len(other_nodes) - 1) looper.run(eventually(check_node_disconnected, disconnected, other_nodes, retryWait=1, timeout=timeout))
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, wallet1, client1): """ Test that a primary election does happen after a view change """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's throughput falls # and view changes nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(10, 0)) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Ensure view change happened for both node and its primary elector for node in nodeSet: looper.run( eventually(partial(checkViewChangeInitiatedForNode, node, 1), retryWait=1, timeout=20)) # Ensure elections are done again and pool is setup again with appropriate # protocol instances and each protocol instance is setup properly too checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, wallet1, client1): """ Checking if average latency is being set """ for i in range(5): req = sendRandomRequest(wallet1, client1) looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for node in nodeSet: # type: Node mLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId, node.instances.masterId) bLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId, *node.instances.backupIds) logger.debug( "Avg. master latency : {}. Avg. backup latency: {}".format( mLat, bLat)) assert mLat > 0 assert bLat > 0
def checkPoolReady(looper: Looper, nodes: Sequence[TestNode], timeout: int = 20): looper.run( eventually(checkNodesAreReady, nodes, retryWait=.25, timeout=timeout, ratchetSteps=10))
def testPostingThroughput(postingStatsEnabled, decreasedMonitoringTimeouts, looper: Looper, nodeSet: TestNodeSet, wallet1, client1): """ The throughput after `DashboardUpdateFreq` seconds and before sending any requests should be zero. Send `n` requests in less than `ThroughputWindowSize` seconds and the throughput till `ThroughputWindowSize` should consider those `n` requests. After `ThroughputWindowSize` seconds the throughput should be zero Test `totalRequests` too. """ config = decreasedMonitoringTimeouts # We are sleeping for this window size, because we need to clear previous # values that were being stored for this much time in tests looper.runFor(config.ThroughputWindowSize) reqCount = 10 for node in nodeSet: assert node.monitor.highResThroughput == 0 assert node.monitor.totalRequests == 0 sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, reqCount, nodeSet.f) for node in nodeSet: assert len(node.monitor.orderedRequestsInLast) == reqCount assert node.monitor.highResThroughput > 0 assert node.monitor.totalRequests == reqCount # TODO: Add implementation to actually call firebase plugin # and test if firebase plugin is sending total request count # if node is primary looper.runFor(config.DashboardUpdateFreq) for node in nodeSet: node.monitor.spylog.count(Monitor.sendThroughput.__name__) > 0 # Run for latency window duration so that `orderedRequestsInLast` # becomes empty looper.runFor(config.ThroughputWindowSize) def chk(): for node in nodeSet: assert len(node.monitor.orderedRequestsInLast) == 0 assert node.monitor.highResThroughput == 0 assert node.monitor.totalRequests == reqCount timeout = config.ThroughputWindowSize looper.run(eventually(chk, retryWait=1, timeout=timeout))
def prepareNodeSet(looper: Looper, txnPoolNodeSet): # TODO: Come up with a more specific name for this # Key sharing party looper.run(checkNodesConnected(txnPoolNodeSet)) # Remove all the nodes for n in list(txnPoolNodeSet): looper.removeProdable(txnPoolNodeSet) txnPoolNodeSet.remove(n)
def reconnect_node_and_ensure_connected(looper: Looper, poolNodes: Iterable[TestNode], connect: Union[str, TestNode], timeout=None): if isinstance(connect, TestNode): connect = connect.name assert isinstance(connect, str) reconnectPoolNode(looper, poolNodes, connect) looper.run(checkNodesConnected(poolNodes, customTimeout=timeout))
def prepareNodeSet(looper: Looper, nodeSet: TestNodeSet): # TODO: Come up with a more specific name for this # Key sharing party looper.run(checkNodesConnected(nodeSet)) # Remove all the nodes for n in list(nodeSet.nodes.keys()): looper.removeProdable(nodeSet.nodes[n]) nodeSet.removeNode(n, shouldClean=False)
def setupNodesAndClient(looper: Looper, nodes: Sequence[TestNode], nodeReg=None, tmpdir=None): looper.run(checkNodesConnected(nodes)) timeout = 15 + 2 * (len(nodes)) ensureElectionsDone(looper=looper, nodes=nodes, retryWait=1, timeout=timeout) return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def checkPoolReady(looper: Looper, nodes: Sequence[TestNode], customTimeout=None): """ Check that pool is in Ready state """ timeout = customTimeout or waits.expectedPoolStartUpTimeout(len(nodes)) looper.run( eventually(checkNodesAreReady, nodes, retryWait=.25, timeout=timeout, ratchetSteps=10))
def checkEveryNodeHasAtMostOnePrimary(looper: Looper, nodes: Sequence[TestNode], retryWait: float = None, customTimeout: float = None): def checkAtMostOnePrim(node): prims = [r for r in node.replicas.values() if r.isPrimary] assert len(prims) <= 1 timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes)) for node in nodes: looper.run(eventually(checkAtMostOnePrim, node, retryWait=retryWait, timeout=timeout))
def checkEveryNodeHasAtMostOnePrimary(looper: Looper, nodes: Sequence[TestNode], retryWait: float = None, customTimeout: float = None): def checkAtMostOnePrim(node): prims = [r for r in node.replicas if r.isPrimary] assert len(prims) <= 1 timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes)) for node in nodes: looper.run(eventually(checkAtMostOnePrim, node, retryWait=retryWait, timeout=timeout))
def setupClient(looper: Looper, nodes: Sequence[TestNode] = None, nodeReg=None, tmpdir=None, identifier=None, verkey=None): client1, wallet = genTestClient(nodes=nodes, nodeReg=nodeReg, tmpdir=tmpdir, identifier=identifier, verkey=verkey) looper.add(client1) looper.run(client1.ensureConnectedToNodes()) return client1, wallet
def checkEveryNodeHasAtMostOnePrimary(looper: Looper, nodes: Sequence[TestNode], retryWait: float = None, timeout: float = None): def checkAtMostOnePrim(node): prims = [r for r in node.replicas if r.isPrimary] assert len(prims) <= 1 for node in nodes: looper.run( eventually(checkAtMostOnePrim, node, retryWait=retryWait, timeout=timeout))
def testPostingLatency(postingStatsEnabled, decreasedMonitoringTimeouts, looper: Looper, nodeSet: TestNodeSet, wallet1, client1): """ The latencies (master as well as average of backups) after `DashboardUpdateFreq` seconds and before sending any requests should be zero. Send `n` requests in less than `LatencyWindowSize` seconds and the latency till `LatencyWindowSize` should consider those `n` requests. After `LatencyWindowSize` seconds the latencies should be zero """ config = decreasedMonitoringTimeouts # Run for latency window duration so that `latenciesByMasterInLast` and # `latenciesByBackupsInLast` become empty looper.runFor(config.LatencyWindowSize) reqCount = 10 for node in nodeSet: assert node.monitor.masterLatency == 0 assert node.monitor.avgBackupLatency == 0 sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, reqCount, nodeSet.f) for node in nodeSet: assert node.monitor.masterLatency > 0 assert node.monitor.avgBackupLatency > 0 looper.runFor(config.DashboardUpdateFreq) for node in nodeSet: node.monitor.spylog.count(Monitor.sendLatencies.__name__) > 0 # Run for latency window duration so that `latenciesByMasterInLast` and # `latenciesByBackupsInLast` become empty looper.runFor(config.LatencyWindowSize) def chk(): for node in nodeSet: assert node.monitor.masterLatency == 0 assert node.monitor.avgBackupLatency == 0 timeout = config.LatencyWindowSize looper.run(eventually(chk, retryWait=1, timeout=timeout))
def wait_for_elections_done_on_given_nodes(looper: Looper, nodes: Iterable[Node], num_of_instances: int, timeout: float, retry_wait: float=1.0): """ Wait for primary elections to be completed on all the replicas of the given nodes. """ def check_num_of_replicas(): for node in nodes: assert len(node.replicas) == num_of_instances def verify_each_replica_knows_its_primary(): for node in nodes: for inst_id, replica in node.replicas.items(): assert replica.hasPrimary looper.run(eventuallyAll(check_num_of_replicas, verify_each_replica_knows_its_primary, totalTimeout=timeout, retryWait=retry_wait))
class UserScenario(metaclass=ABCMeta): def __init__(self, seed, logFileName=None): if logFileName: Logger().enableFileLogging(logFileName) self._seed = seed self._client = None self._wallet = None self._looper = None @property def identifier(self): if self._wallet: return self._wallet.defaultId else: return None @property def verkey(self): if self._wallet: return self._wallet.getVerkey() else: return None @classmethod def runInstance(cls, *args, **kwargs): cls(*args, **kwargs).run() def run(self): try: self._createClientAndWallet() self._looper = Looper(debug=getConfig().LOOPER_DEBUG) try: self._startClient() self.do() finally: self._looper.shutdownSync() self._looper = None except BaseException as ex: logger.exception( "User scenario throws out exception: {}".format(ex), exc_info=ex) raise ex @abstractmethod def do(self): pass def performOperation(self, op): req = self._wallet.signOp(op) self._client.submitReqs(req) def getRequestResult(reqKey): reply, error = self._client.replyIfConsensus(*reqKey) if reply is None and error is None: raise Exception("Request has not been completed yet") else: return reply, error reply, error = self._looper.run( eventually(partial(getRequestResult, req.key), retryWait=.5, timeout=5)) assert not error, error if reply[DATA]: result = json.loads(reply[DATA]) else: result = None return result def generateNewSigner(self): assert self.identifier return SimpleSigner(identifier=self.identifier) def changeSigner(self, newSigner): assert newSigner.identifier == self.identifier self._wallet.updateSigner(self.identifier, newSigner) logger.info("Changed signer. New verkey: {}".format(self.verkey)) def _createClientAndWallet(self): signer = SimpleSigner(seed=self._seed) port = genHa()[1] ha = HA('0.0.0.0', port) self._client = Client(name=signer.identifier, ha=ha) self._wallet = Wallet(name=signer.identifier) self._wallet.addIdentifier(signer=signer) logger.info("Identifier: {}".format(self.identifier)) logger.info("Signer's verkey: {}".format(self.verkey)) def _startClient(self): self._looper.add(self._client) def ensureConnectedToAll(): connectedNodes = self._client.nodestack.connecteds connectedNodesNum = len(connectedNodes) totalNodes = len(self._client.nodeReg) logger.info("Connected {} / {} nodes".format( connectedNodesNum, totalNodes)) for node in connectedNodes: logger.info(" {}".format(node)) if connectedNodesNum == 0: raise Exception("Not connected to any") elif connectedNodesNum < totalNodes * 0.8: raise Exception("Not connected fully") else: return True self._looper.run( eventually(ensureConnectedToAll, retryWait=.5, timeout=5))