def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) n = 250 timeOutPerReq = 3 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, n)) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, n, 1, timeOutPerReq) logger.debug("{} sent {} requests".format(client, n)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}".format( node, len(node.requests), get_size(node.requests))) for replica in node.replicas: logger.debug("{} has prepares {} with size {}".format( replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}".format( replica, len(replica.commits), get_size(replica.commits)))
def testNodePortChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded): """ An running node's port is changed """ newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) newNode.stop() looper.removeProdable(name=newNode.name) logger.debug("{} starting with HAs {} {}".format(newNode, nodeNewHa, clientNewHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): """ A client should be able to connect to a newly added node """ newStewardName = "testClientSteward" + randomString(3) newNodeName = "Epsilon" oldNodeReg = copy(steward1.nodeReg) newSteward, newStewardWallet, newNode = addNewStewardAndNode( looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) def chkNodeRegRecvd(): assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1 assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=5)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): """ A client should be able to connect to a newly added node """ newStewardName = "testClientSteward"+randomString(3) newNodeName = "Epsilon" oldNodeReg = copy(steward1.nodeReg) newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) def chkNodeRegRecvd(): assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1 assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=5)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testNodeKeysChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, nodeThetaAdded, allPluginsPath=None): newSteward, newStewardWallet, newNode = nodeThetaAdded newNode.stop() looper.removeProdable(name=newNode.name) nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha) sigseed = randomString(32).encode() verkey = base58.b58encode(SimpleSigner(seed=sigseed).naclSigner.verraw) changeNodeKeys(looper, newSteward, newStewardWallet, newNode, verkey) initNodeKeysForBothStacks(newNode.name, tdirWithPoolTxns, sigseed, override=True) logger.debug("{} starting with HAs {} {}".format(newNode, nodeHa, nodeCHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(stacks=txnPoolNodeSet)) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too # name, seed = poolTxnClientData # signer = SimpleSigner(seed=seed) # name = "testClient96541" # ha = genHa() # client = genTestClient(txnPoolNodeSet, signer=signer, ha=ha, # tmpdir=tdirWithPoolTxns, usePoolLedger=True, # name=name) client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) basedirpath = client.basedirpath looper.removeProdable(client) # Removing RAET keep directory otherwise the client will use the same port # since it will a directory of its name in the keep shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) n = 250 timeOutPerReq = 3 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, n)) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, n, 1, timeOutPerReq) logger.debug("{} sent {} requests".format(client, n)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}". format(node, len(node.requests), get_size(node.requests))) for replica in node.replicas: logger.debug("{} has prepares {} with size {}". format(replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}". format(replica, len(replica.commits), get_size(replica.commits)))
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) numRequests = 250 fVal = 1 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, numRequests)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, numRequests) logger.debug("{} sent {} requests".format(client, numRequests)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}".format( node, len(node.requests), get_size(node.requests))) for replica in node.replicas.values(): logger.debug("{} has prepares {} with size {}".format( replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}".format( replica, len(replica.commits), get_size(replica.commits)))
def add_started_node(looper, new_node, node_ha, client_ha, txnPoolNodeSet, client_tdir, stewardClient, stewardWallet, sigseed, bls_key): ''' Adds already created node to the pool, that is sends NODE txn. Makes sure that node is actually added and connected to all otehr nodes. ''' newSteward, newStewardWallet = addNewSteward(looper, client_tdir, stewardClient, stewardWallet, "Steward" + new_node.name, clientClass=TestClient) node_name = new_node.name send_new_node_txn(sigseed, node_ha[0], node_ha[1], client_ha[0], client_ha[1], bls_key, node_name, newSteward, newStewardWallet) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, tdirWithClientPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) keys_dir = os.path.join(client.keys_dir, client.name) client.stop() looper.removeProdable(client) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithClientPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def nodeThetaAdded(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath, testNodeClass=None, testClientClass=None, name=None): newStewardName = "testClientSteward" + randomString(3) newNodeName = name or "Theta" newSteward, newStewardWallet, newNode = addNewStewardAndNode( looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath, nodeClass=testNodeClass, clientClass=testClientClass) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) return newSteward, newStewardWallet, newNode
def new_client(looper, poolTxnClientData, txnPoolNodeSet, client_tdir): client, wallet = buildPoolClientAndWallet(poolTxnClientData, client_tdir) looper.add(client) looper.run(client.ensureConnectedToNodes()) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) return client, wallet
def testNodeKeysChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, nodeThetaAdded, allPluginsPath=None): newSteward, newStewardWallet, newNode = nodeThetaAdded # Since the node returned by fixture `nodeThetaAdded` was abandoned in the # previous test, so getting node `Theta` from `txnPoolNodeSet` newNode = getNodeWithName(txnPoolNodeSet, newNode.name) newNode.stop() nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha) sigseed = randomString(32).encode() verkey = SimpleSigner(seed=sigseed).naclSigner.verhex.decode() changeNodeKeys(looper, newSteward, newStewardWallet, newNode, verkey) initLocalKeep(newNode.name, tdirWithPoolTxns, sigseed) initLocalKeep(newNode.name+CLIENT_STACK_SUFFIX, tdirWithPoolTxns, sigseed) looper.removeProdable(name=newNode.name) logger.debug("{} starting with HAs {} {}".format(newNode, nodeHa, nodeCHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) basedirpath = client.basedirpath looper.removeProdable(client) # Removing RAET keep directory otherwise the client will use the same port # since it will a directory of its name in the keep shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def testNodePortChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded): """ An running node's port is changed """ newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa = genHa(1) new_port = nodeNewHa.port node_ha = txnPoolNodeSet[0].nodeReg[newNode.name] cli_ha = txnPoolNodeSet[0].cliNodeReg[newNode.name + CLIENT_STACK_SUFFIX] node_data = { ALIAS: newNode.name, NODE_PORT: new_port, NODE_IP: node_ha.host, CLIENT_PORT: cli_ha.port, CLIENT_IP: cli_ha.host, } node = updateNodeDataAndReconnect(looper, newSteward, newStewardWallet, newNode, node_data, tdirWithPoolTxns, tconf, txnPoolNodeSet) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def test_add_node_to_pool_with_large_ppseqno_diff_views( do_view_change, looper, txnPoolNodeSet, tconf, steward1, stewardWallet, tdir, client_tdir, allPluginsPath): """ Adding a node to the pool while ppSeqNo is big caused a node to stash all the requests because of incorrect watermarks limits set. The case of view_no == 0 is special. The test emulates big ppSeqNo number, adds a node and checks all the pool nodes are functional. The test is run with several starting view_no, including 0 """ # TODO: for now this test will use old client api, after moving node txn to sdk it will be rewritten ensure_several_view_change(looper, txnPoolNodeSet, do_view_change, custom_timeout=tconf.VIEW_CHANGE_TIMEOUT) big_ppseqno = tconf.LOG_SIZE * 2 + 2345 cur_ppseqno = _get_ppseqno(txnPoolNodeSet) assert (big_ppseqno > cur_ppseqno) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) _set_ppseqno(txnPoolNodeSet, big_ppseqno) cur_ppseqno = _get_ppseqno(txnPoolNodeSet) assert (big_ppseqno == cur_ppseqno) sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) new_steward_name = "testClientSteward" + randomString(4) new_node_name = "TestTheta" + randomString(4) new_steward, new_steward_wallet, new_node = \ addNewStewardAndNode(looper, steward1, stewardWallet, new_steward_name, new_node_name, tdir, client_tdir, tconf, allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, new_steward, *txnPoolNodeSet) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) looper.run( eventually(checkNodeLedgersForEquality, node, *restartedNodes[:-1], retryWait=1, timeout=10)) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def testClientUsingPoolTxns(looper, txnPoolNodeSet, poolTxnClient): """ Client should not be using node registry but pool transaction file :return: """ client, wallet = poolTxnClient looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def client_and_wallet(txnPoolNodesLooper, tdirWithPoolTxns, txnPoolNodeSet): client, wallet = genTestClient(tmpdir=tdirWithPoolTxns, nodes=txnPoolNodeSet, name='reader', usePoolLedger=True) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) return client, wallet
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tdirWithPoolTxns, tdirWithClientPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded op = { ALIAS: newNode.name, NODE_IP: nodeNewHa.host, NODE_PORT: nodeNewHa.port, CLIENT_IP: clientNewHa.host, CLIENT_PORT: clientNewHa.port, } updateNodeData(looper, newSteward, newStewardWallet, newNode, op) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir) node = TestNode(newNode.name, config_helper=config_helper, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, steward, stewardWallet, allPluginsPath, testNodeClass, testClientClass, tdir): newStewardName = "testClientSteward" + randomString(3) newNodeName = "Theta" newSteward, newStewardWallet = getClientAddedWithRole(nodeSet, tdir, looper, steward, stewardWallet, newStewardName, role=STEWARD) sigseed = randomString(32).encode() nodeSigner = SimpleSigner(seed=sigseed) (nodeIp, nodePort), (clientIp, clientPort) = genHa(2) data = { NODE_IP: nodeIp, NODE_PORT: nodePort, CLIENT_IP: clientIp, CLIENT_PORT: clientPort, ALIAS: newNodeName, SERVICES: [VALIDATOR, ] } node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId) newStewardWallet.addNode(node) reqs = newStewardWallet.preparePending() req, = newSteward.submitReqs(*reqs) waitForSufficientRepliesForRequests(looper, newSteward, requests=[req]) def chk(): assert newStewardWallet.getNode(node.id).seqNo is not None timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) initNodeKeysForBothStacks(newNodeName, tdirWithPoolTxns, sigseed, override=True) newNode = testNodeClass(newNodeName, basedirpath=tdir, config=tconf, ha=(nodeIp, nodePort), cliha=(clientIp, clientPort), pluginPaths=allPluginsPath) nodeSet.append(newNode) looper.add(newNode) looper.run(checkNodesConnected(nodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, *nodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *nodeSet) return newSteward, newStewardWallet, newNode
def nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, steward, stewardWallet, allPluginsPath, testNodeClass, testClientClass, tdir): newStewardName = "testClientSteward" + randomString(3) newNodeName = "Theta" newSteward, newStewardWallet = getClientAddedWithRole( nodeSet, tdir, looper, steward, stewardWallet, newStewardName, STEWARD) sigseed = randomString(32).encode() nodeSigner = SimpleSigner(seed=sigseed) (nodeIp, nodePort), (clientIp, clientPort) = genHa(2) data = { NODE_IP: nodeIp, NODE_PORT: nodePort, CLIENT_IP: clientIp, CLIENT_PORT: clientPort, ALIAS: newNodeName, SERVICES: [ VALIDATOR, ] } node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId) newStewardWallet.addNode(node) reqs = newStewardWallet.preparePending() req, = newSteward.submitReqs(*reqs) checkSufficientRepliesForRequests(looper, newSteward, [ req, ]) def chk(): assert newStewardWallet.getNode(node.id).seqNo is not None looper.run(eventually(chk, retryWait=1, timeout=10)) initLocalKeep(newNodeName, tdirWithPoolTxns, sigseed, override=True) newNode = testNodeClass(newNodeName, basedirpath=tdir, config=tconf, ha=(nodeIp, nodePort), cliha=(clientIp, clientPort), pluginPaths=allPluginsPath) nodeSet.append(newNode) looper.add(newNode) looper.run(checkNodesConnected(nodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, *nodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *nodeSet) return newSteward, newStewardWallet, newNode
def testClientConnectToRestartedNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientNames, poolTxnData, tconf, poolTxnNodeNames, allPluginsPath): name = poolTxnClientNames[-1] seed = poolTxnData["seeds"][name] newClient, w = genTestClient(tmpdir=tdirWithPoolTxns, nodes=txnPoolNodeSet, name=name, usePoolLedger=True) looper.add(newClient) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 1, 1) for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # looper.run(newClient.ensureDisconnectedToNodes(timeout=60)) txnPoolNodeSet = [] for nm in poolTxnNodeNames: node = TestNode(nm, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(node) txnPoolNodeSet.append(node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, retryWait=1, timeout=10) def chk(): for node in txnPoolNodeSet: assert node.isParticipating looper.run(eventually(chk, retryWait=1, timeout=10)) bootstrapClientKeys(w.defaultId, w.getVerkey(), txnPoolNodeSet) req = sendRandomRequest(w, newClient) checkSufficientRepliesForRequests(looper, newClient, [ req, ], timeoutPerReq=10) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 1, 1)
def stewards_and_wallets(looper, txnPoolNodeSet, pool_txn_stewards_data, tdirWithPoolTxns): clients_and_wallets = [] for pool_txn_steward_data in pool_txn_stewards_data: steward_client, steward_wallet = buildPoolClientAndWallet( pool_txn_steward_data, tdirWithPoolTxns) looper.add(steward_client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client, *txnPoolNodeSet) clients_and_wallets.append((steward_client, steward_wallet)) yield clients_and_wallets for (client, wallet) in clients_and_wallets: client.stop()
def testClientConnectToRestartedNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tdir, tdirWithClientPoolTxns, poolTxnClientNames, poolTxnData, tconf, poolTxnNodeNames, allPluginsPath): name = poolTxnClientNames[-1] newClient, w = genTestClient(tmpdir=tdirWithClientPoolTxns, nodes=txnPoolNodeSet, name=name, usePoolLedger=True) looper.add(newClient) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 1, 1) for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # looper.run(newClient.ensureDisconnectedToNodes(timeout=60)) txnPoolNodeSet = [] for nm in poolTxnNodeNames: config_helper = PNodeConfigHelper(nm, tconf, chroot=tdir) node = TestNode(nm, ledger_dir=config_helper.ledger_dir, keys_dir=config_helper.keys_dir, genesis_dir=config_helper.genesis_dir, plugins_dir=config_helper.plugins_dir, config=tconf, pluginPaths=allPluginsPath) looper.add(node) txnPoolNodeSet.append(node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) def chk(): for node in txnPoolNodeSet: assert node.isParticipating timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) bootstrapClientKeys(w.defaultId, w.getVerkey(), txnPoolNodeSet) req = sendRandomRequest(w, newClient) waitForSufficientRepliesForRequests(looper, newClient, requests=[req]) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 3, 1)
def change_bls_key(looper, txnPoolNodeSet, tdirWithPoolTxns, node, steward_client, steward_wallet, add_wrong=False): new_blspk = init_bls_keys(tdirWithPoolTxns, node.name) key_in_txn = new_blspk if not add_wrong else randomString(32) node_data = {ALIAS: node.name, BLS_KEY: key_in_txn} updateNodeData(looper, steward_client, steward_wallet, node, node_data) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client, *txnPoolNodeSet) return new_blspk
def nodeStashingOrderedRequests(txnPoolNodeSet, nodeCreatedAfterSomeTxns): looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns for node in txnPoolNodeSet: node.nodeIbStasher.delay(crDelay(5)) txnPoolNodeSet.append(newNode) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet[:-1]) sendRandomRequests(wallet, client, 10) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=15)) def stashing(): assert newNode.mode != Mode.participating assert len(newNode.stashedOrderedReqs) > 0 assert len(newNode.reqsFromCatchupReplies) > 0 looper.run(eventually(stashing, retryWait=1, timeout=20))
def testNodeKeysChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, nodeThetaAdded, allPluginsPath=None): newSteward, newStewardWallet, newNode = nodeThetaAdded # Since the node returned by fixture `nodeThetaAdded` was abandoned in the # previous test, so getting node `Theta` from `txnPoolNodeSet` newNode = getNodeWithName(txnPoolNodeSet, newNode.name) newNode.stop() looper.removeProdable(name=newNode.name) nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha) sigseed = randomString(32).encode() verkey = SimpleSigner(seed=sigseed).naclSigner.verhex.decode() changeNodeKeys(looper, newSteward, newStewardWallet, newNode, verkey) initNodeKeysForBothStacks(newNode.name, tdirWithPoolTxns, sigseed, override=True) logger.debug("{} starting with HAs {} {}".format(newNode, nodeHa, nodeCHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=40)) looper.run( eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) looper.run(eventually(checkNodeLedgersForEquality, node, *restartedNodes[:-1], retryWait=1, timeout=10)) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def nodeThetaAdded(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): newStewardName = "testClientSteward" + randomString(3) newNodeName = "Theta" newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) return newSteward, newStewardWallet, newNode
def testClientConnectToRestartedNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientNames, poolTxnData, tconf, poolTxnNodeNames, allPluginsPath): name = poolTxnClientNames[-1] seed = poolTxnData["seeds"][name] newClient, w = genTestClient(tmpdir=tdirWithPoolTxns, nodes=txnPoolNodeSet, name=name, usePoolLedger=True) looper.add(newClient) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 1, 1) for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # looper.run(newClient.ensureDisconnectedToNodes(timeout=60)) txnPoolNodeSet = [] for nm in poolTxnNodeNames: node = TestNode(nm, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(node) txnPoolNodeSet.append(node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, retryWait=1, timeout=10) def chk(): for node in txnPoolNodeSet: assert node.isParticipating looper.run(eventually(chk, retryWait=1, timeout=10)) bootstrapClientKeys(w.defaultId, w.getVerkey(), txnPoolNodeSet) req = sendRandomRequest(w, newClient) checkSufficientRepliesForRequests(looper, newClient, [req, ], timeoutPerReq=10) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newClient, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, w, newClient, 1, 1)
def change_bls_key(looper, txnPoolNodeSet, node, steward_client, steward_wallet, add_wrong=False): new_blspk = init_bls_keys(node.keys_dir, node.name) key_in_txn = \ new_blspk \ if not add_wrong \ else ''.join(random_from_alphabet(32, base58.alphabet)) node_data = {ALIAS: node.name, BLS_KEY: key_in_txn} updateNodeData(looper, steward_client, steward_wallet, node, node_data) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client, *txnPoolNodeSet) return new_blspk
def testNodePortChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded): """ An running node's port is changed """ newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) newNode.stop() looper.removeProdable(name=newNode.name) logger.debug("{} starting with HAs {} {}".format(newNode, nodeNewHa, clientNewHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run( eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def steward1(looper, txnPoolNodeSet, stewardAndWallet1): steward, wallet = stewardAndWallet1 looper.add(steward) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, *txnPoolNodeSet) return steward
def testNodesReceiveClientMsgs(looper, txnPoolNodeSet, wallet1, client1, client1Connected): ensureClientConnectedToNodesAndPoolLedgerSame(looper, client1, *txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
def nodeThetaAdded(looper, nodeSet, tdirWithClientPoolTxns, tconf, steward, stewardWallet, allPluginsPath, testNodeClass, testClientClass, node_config_helper_class, tdir, node_name='Theta'): newStewardName = "testClientSteward" + randomString(3) newNodeName = node_name newSteward, newStewardWallet = getClientAddedWithRole( nodeSet, tdirWithClientPoolTxns, looper, steward, stewardWallet, newStewardName, role=STEWARD) sigseed = randomString(32).encode() nodeSigner = SimpleSigner(seed=sigseed) (nodeIp, nodePort), (clientIp, clientPort) = genHa(2) config_helper = node_config_helper_class(newNodeName, tconf, chroot=tdir) _, _, bls_key = initNodeKeysForBothStacks(newNodeName, config_helper.keys_dir, sigseed, override=True) data = { NODE_IP: nodeIp, NODE_PORT: nodePort, CLIENT_IP: clientIp, CLIENT_PORT: clientPort, ALIAS: newNodeName, SERVICES: [ VALIDATOR, ], BLS_KEY: bls_key } node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId) newStewardWallet.addNode(node) reqs = newStewardWallet.preparePending() req = newSteward.submitReqs(*reqs)[0][0] waitForSufficientRepliesForRequests(looper, newSteward, requests=[req]) def chk(): assert newStewardWallet.getNode(node.id).seqNo is not None timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) newNode = testNodeClass(newNodeName, config_helper=config_helper, config=tconf, ha=(nodeIp, nodePort), cliha=(clientIp, clientPort), pluginPaths=allPluginsPath) nodeSet.append(newNode) looper.add(newNode) looper.run(checkNodesConnected(nodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, *nodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *nodeSet) return newSteward, newStewardWallet, newNode
def testStewardSuspendsNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnStewardData, allPluginsPath): newSteward, newStewardWallet, newNode = nodeThetaAdded newNodeNym = hexToFriendly(newNode.nodestack.local.signer.verhex) suspendNode(looper, newSteward, newStewardWallet, newNodeNym, newNode.name) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: looper.run(eventually(checkNodeNotInNodeReg, node, newNode.name)) for client in (steward1, newSteward): looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name)) # Check a client can send request and receive replies req = sendRandomRequest(newStewardWallet, newSteward) checkSufficientRepliesForRequests(looper, newSteward, [ req, ], timeoutPerReq=10) # Check that a restarted client or node does not connect to the suspended # node steward1.stop() looper.removeProdable(steward1) steward1, stewardWallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns) looper.add(steward1) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) looper.run(eventually(checkNodeNotInNodeReg, steward1, newNode.name)) newNode.stop() looper.removeProdable(newNode) # TODO: There is a bug that if a primary node is turned off, it sends # duplicate Pre-Prepare and gets blacklisted. Here is the gist # https://gist.github.com/lovesh/c16989616ebb6856f9fa2905c14dc4b7 oldNodeIdx, oldNode = [(i, n) for i, n in enumerate(txnPoolNodeSet) if not n.hasPrimary][0] oldNode.stop() looper.removeProdable(oldNode) oldNode = TestNode(oldNode.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(oldNode) txnPoolNodeSet[oldNodeIdx] = oldNode looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeNotInNodeReg, oldNode, newNode.name)) # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node cancelNodeSuspension(looper, newSteward, newStewardWallet, newNodeNym, newNode.name) nodeTheta = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) looper.add(nodeTheta) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=30)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)