def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, 'AnotherSteward' + randomString(4), 'AnotherNode' + randomString(4), tdir, tconf) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) node_new_ha, client_new_ha = genHa(2) logger.debug("{} changing HAs to {} {}".format(new_node, node_new_ha, client_new_ha)) # Making the change HA txn an confirming its succeeded node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, node_new_ha.host, node_new_ha.port, client_new_ha.host, client_new_ha.port) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) node = TestNode(new_node.name, config_helper=config_helper, config=tconf, ha=node_new_ha, cliha=client_new_ha) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, restartedNodes, sdk_wallet_client, sdk_pool_handle)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tdirWithPoolTxns, tdirWithClientPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded op = { ALIAS: newNode.name, NODE_IP: nodeNewHa.host, NODE_PORT: nodeNewHa.port, CLIENT_IP: clientNewHa.host, CLIENT_PORT: clientNewHa.port, } updateNodeData(looper, newSteward, newStewardWallet, newNode, op) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir) node = TestNode(newNode.name, config_helper=config_helper, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def testNodeKeysChanged(looper, txnPoolNodeSet, tdir, tconf, sdk_node_theta_added, sdk_pool_handle, allPluginsPath=None): new_steward_wallet, new_node = sdk_node_theta_added new_node.stop() looper.removeProdable(name=new_node.name) nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha) sigseed = randomString(32).encode() verkey = base58.b58encode(SimpleSigner(seed=sigseed).naclSigner.verraw).decode("utf-8") sdk_change_node_keys(looper, new_node, new_steward_wallet, sdk_pool_handle, verkey) config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) initNodeKeysForBothStacks(new_node.name, config_helper.keys_dir, sigseed, override=True) logger.debug("{} starting with HAs {} {}".format(new_node, nodeHa, nodeCHa)) node = TestNode(new_node.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
def addNode(self, name: str) -> TestNode: if name in self.nodes: error("{} already added".format(name)) assert name in self.nodeReg ha, cliname, cliha = self.nodeReg[name] config_helper = PNodeConfigHelper(name, self.config, chroot=self.tmpdir) seed = randomSeed() if self.keyshare: learnKeysFromOthers(config_helper.keys_dir, name, self.nodes.values()) testNodeClass = self.testNodeClass node = self.enter_context( testNodeClass(name=name, ha=ha, cliname=cliname, cliha=cliha, config_helper=config_helper, primaryDecider=self.primaryDecider, pluginPaths=self.pluginPaths, seed=seed)) if self.keyshare: tellKeysToOthers(node, self.nodes.values()) self.nodes[name] = node self.__dict__[name] = node return node
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, start_one_by_one=True, wait_for_elections=True): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set.copy(): config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node restart_set[idx] = restarted_node rest_nodes += [restarted_node] if start_one_by_one: looper.run(checkNodesConnected(rest_nodes)) if not start_one_by_one: looper.run(checkNodesConnected(nodeSet)) if wait_for_elections: ensureElectionsDone(looper=looper, nodes=nodeSet)
def update_node_data_and_reconnect(looper, txnPoolNodeSet, steward_wallet, sdk_pool_handle, node, new_node_ip, new_node_port, new_client_ip, new_client_port, tdir, tconf): node_ha = node.nodestack.ha cli_ha = node.clientstack.ha node_dest = hexToFriendly(node.nodestack.verhex) sdk_send_update_node(looper, steward_wallet, sdk_pool_handle, node_dest, node.name, new_node_ip, new_node_port, new_client_ip, new_client_port) # restart the Node with new HA node.stop() looper.removeProdable(name=node.name) config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=HA(new_node_ip or node_ha.host, new_node_port or node_ha.port), cliha=HA(new_client_ip or cli_ha.host, new_client_port or cli_ha.port)) looper.add(restartedNode) # replace node in txnPoolNodeSet try: idx = next(i for i, n in enumerate(txnPoolNodeSet) if n.name == node.name) except StopIteration: raise Exception('{} is not the pool'.format(node)) txnPoolNodeSet[idx] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_ensure_pool_functional(looper, txnPoolNodeSet, steward_wallet, sdk_pool_handle) return restartedNode
def test_state_regenerated_from_ledger(looper, txnPoolNodeSet, client1, wallet1, client1Connected, tdir, tconf, allPluginsPath): """ Node loses its state database but recreates it from ledger after start """ sent_batches = 10 send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 5 * sent_batches, sent_batches) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) node_to_stop = txnPoolNodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path nodeHa, nodeCHa = HA(*node_to_stop.nodestack.ha), HA( *node_to_stop.clientstack.ha) node_to_stop.stop() looper.removeProdable(node_to_stop) shutil.rmtree(state_db_path) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(restarted_node) txnPoolNodeSet[-1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, restarted_node, *txnPoolNodeSet[:-1])
def updateNodeDataAndReconnect(looper, steward, stewardWallet, node, node_data, tdir, tconf, txnPoolNodeSet): updateNodeData(looper, steward, stewardWallet, node, node_data) # restart the Node with new HA node.stop() node_alias = node_data.get(ALIAS, None) or node.name node_ip = node_data.get(NODE_IP, None) or node.nodestack.ha.host node_port = node_data.get(NODE_PORT, None) or node.nodestack.ha.port client_ip = node_data.get(CLIENT_IP, None) or node.clientstack.ha.host client_port = node_data.get(CLIENT_PORT, None) or node.clientstack.ha.port looper.removeProdable(name=node.name) config_helper = PNodeConfigHelper(node_alias, tconf, chroot=tdir) restartedNode = TestNode(node_alias, config_helper=config_helper, config=tconf, ha=HA(node_ip, node_port), cliha=HA(client_ip, client_port)) looper.add(restartedNode) # replace node in txnPoolNodeSet try: idx = next(i for i, n in enumerate(txnPoolNodeSet) if n.name == node.name) except StopIteration: raise Exception('{} is not the pool'.format(node)) txnPoolNodeSet[idx] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet)) return restartedNode
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): looper = looper_without_nodeset_for_func nodes = [] initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) for name in nodeReg: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper)
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, per_add_timeout=None): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set: config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node if per_add_timeout: looper.run(checkNodesConnected(rest_nodes + [restarted_node], customTimeout=per_add_timeout)) rest_nodes += [restarted_node] if not per_add_timeout: looper.run(checkNodesConnected(nodeSet, customTimeout=after_restart_timeout))
def test_order_after_demote_and_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath, sdk_wallet_stewards): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3, 3) primary_node = txnPoolNodeSet[0] node_to_stop = txnPoolNodeSet[1] node_to_demote = txnPoolNodeSet[2] txnPoolNodeSet.remove(node_to_demote) node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet, timeout=2) demote_node(looper, sdk_wallet_stewards[2], sdk_pool_handle, node_to_demote) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, check_primaries=False) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1, 1) def get_current_bls_keys(node): return node.master_replica._bls_bft_replica._bls_bft.bls_key_register._current_bls_keys assert get_current_bls_keys(restarted_node) == get_current_bls_keys(primary_node)
def testClientConnectToRestartedNodes(looper, txnPoolNodeSet, tdir, tconf, poolTxnNodeNames, allPluginsPath, sdk_wallet_new_client, sdk_pool_handle): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_new_client, 1) for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) txnPoolNodeSet = [] for nm in poolTxnNodeNames: config_helper = PNodeConfigHelper(nm, tconf, chroot=tdir) node = TestNode(nm, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath) looper.add(node) txnPoolNodeSet.append(node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) def chk(): for node in txnPoolNodeSet: assert node.isParticipating timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_new_client, sdk_pool_handle)
def testNodeCatchupFPlusOne(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, tdirWithPoolTxns, allPluginsPath, testNodeClass): """ Check that f+1 nodes is enough for catchup """ assert len(txnPoolNodeSet) == 4 node1 = txnPoolNodeSet[-1] node0 = txnPoolNodeSet[-2] logger.debug("Stopping node0 with pool ledger size {}".format( node0.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node0, stopNode=True) looper.removeProdable(node0) logger.debug("Sending requests") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) logger.debug("Stopping node1 with pool ledger size {}".format( node1.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node1, stopNode=True) looper.removeProdable(node1) # Make sure new node got out of sync # Excluding state check since the node is stopped hence the state db is closed waitNodeDataInequality(looper, node0, *txnPoolNodeSet[:-2], exclude_from_check=['check_state']) # TODO: Check if the node has really stopped processing requests? logger.debug("Starting the stopped node0") nodeHa, nodeCHa = HA(*node0.nodestack.ha), HA(*node0.clientstack.ha) config_helper = PNodeConfigHelper(node0.name, tconf, chroot=tdir) node0 = testNodeClass(node0.name, config_helper=config_helper, ha=nodeHa, cliha=nodeCHa, config=tconf, pluginPaths=allPluginsPath) looper.add(node0) logger.debug("Waiting for the node0 to catch up") waitNodeDataEquality(looper, node0, *txnPoolNodeSet[:-2]) logger.debug("Sending more requests") sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) checkNodeDataForEquality(node0, *txnPoolNodeSet[:-2])
def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, tconf, shouldBePrimary, tdir, sdk_pool_handle, sdk_wallet_stewards): # prepare new ha for node and client stack subjectedNode = None node_index = None for nodeIndex, n in enumerate(txnPoolNodeSet): if shouldBePrimary == n.has_master_primary: subjectedNode = n node_index = nodeIndex break nodeStackNewHA, clientStackNewHA = genHa(2) logger.debug("change HA for node: {} to {}".format( subjectedNode.name, (nodeStackNewHA, clientStackNewHA))) # change HA sdk_wallet_steward = sdk_wallet_stewards[node_index] node_dest = hexToFriendly(subjectedNode.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, subjectedNode.name, nodeStackNewHA[0], nodeStackNewHA[1], clientStackNewHA[0], clientStackNewHA[1], services=[VALIDATOR]) # stop node for which HA will be changed subjectedNode.stop() looper.removeProdable(subjectedNode) # start node with new HA config_helper = PNodeConfigHelper(subjectedNode.name, tconf, chroot=tdir) restartedNode = TestNode(subjectedNode.name, config_helper=config_helper, config=tconf, ha=nodeStackNewHA, cliha=clientStackNewHA) looper.add(restartedNode) txnPoolNodeSet[nodeIndex] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet, customTimeout=70)) sdk_pool_refresh(looper, sdk_pool_handle) electionTimeout = waits.expectedPoolElectionTimeout( nodeCount=len(txnPoolNodeSet), numOfReelections=3) ensureElectionsDone(looper, txnPoolNodeSet, retryWait=1, customTimeout=electionTimeout) # start client and check the node HA anotherClient, _ = genTestClient(tmpdir=tdirWithClientPoolTxns, usePoolLedger=True) looper.add(anotherClient) looper.run(eventually(anotherClient.ensureConnectedToNodes)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 8)
def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary, tdir): # prepare new ha for node and client stack subjectedNode = None stewardName = None stewardsSeed = None for nodeIndex, n in enumerate(txnPoolNodeSet): if shouldBePrimary == n.has_master_primary: subjectedNode = n stewardName = poolTxnStewardNames[nodeIndex] stewardsSeed = poolTxnData["seeds"][stewardName].encode() break nodeStackNewHA, clientStackNewHA = genHa(2) logger.debug("change HA for node: {} to {}".format( subjectedNode.name, (nodeStackNewHA, clientStackNewHA))) nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode() # change HA stewardClient, req = changeHA(looper, tconf, subjectedNode.name, nodeSeed, nodeStackNewHA, stewardName, stewardsSeed, basedir=tdirWithClientPoolTxns) waitForSufficientRepliesForRequests(looper, stewardClient, requests=[req]) # stop node for which HA will be changed subjectedNode.stop() looper.removeProdable(subjectedNode) # start node with new HA config_helper = PNodeConfigHelper(subjectedNode.name, tconf, chroot=tdir) restartedNode = TestNode(subjectedNode.name, config_helper=config_helper, config=tconf, ha=nodeStackNewHA, cliha=clientStackNewHA) looper.add(restartedNode) txnPoolNodeSet[nodeIndex] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet, customTimeout=70)) electionTimeout = waits.expectedPoolElectionTimeout( nodeCount=len(txnPoolNodeSet), numOfReelections=3) ensureElectionsDone(looper, txnPoolNodeSet, retryWait=1, customTimeout=electionTimeout) # start client and check the node HA anotherClient, _ = genTestClient(tmpdir=tdirWithClientPoolTxns, usePoolLedger=True) looper.add(anotherClient) looper.run(eventually(anotherClient.ensureConnectedToNodes)) stewardWallet = Wallet(stewardName) stewardWallet.addIdentifier(signer=DidSigner(seed=stewardsSeed)) sendReqsToNodesAndVerifySuffReplies( looper, stewardWallet, stewardClient, 8)
def prepare_new_node_data(tconf, tdir, newNodeName): sigseed = randomString(32).encode() (nodeIp, nodePort), (clientIp, clientPort) = genHa(2) config_helper = PNodeConfigHelper(newNodeName, tconf, chroot=tdir) _, verkey, bls_key = initNodeKeysForBothStacks(newNodeName, config_helper.keys_dir, sigseed, override=True) return sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort
def create(name): config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) return node
def test_valid_txn_with_fees(helpers, mint_tokens, fees_set, nodeSetWithIntegratedTokenPlugin, looper, address_main, addresses, tdir, tconf): seq_no = get_seq_no(mint_tokens) remaining = 1000 last_node = nodeSetWithIntegratedTokenPlugin[-1] last_node.stop() looper.removeProdable(last_node) token_req_handler = last_node.get_req_handler(TOKEN_LEDGER_ID) token_req_handler.utxo_cache._store.close() nodeSetWithIntegratedTokenPlugin = nodeSetWithIntegratedTokenPlugin[:-1] for address in addresses: inputs = [ { ADDRESS: address_main, SEQNO: seq_no }, ] outputs = [ { ADDRESS: address, AMOUNT: 1 }, { ADDRESS: address_main, AMOUNT: remaining - 2 }, # XFER fee is 1 ] request = helpers.request.transfer(inputs, outputs) response = helpers.sdk.send_and_check_request_objects([request]) result = helpers.sdk.get_first_result(response) seq_no = get_seq_no(result) remaining -= 2 for _ in range(5): pay_fees(helpers, fees_set, address_main) config_helper = PNodeConfigHelper(last_node.name, tconf, chroot=tdir) restarted_node = TestNode(last_node.name, config_helper=config_helper, config=tconf, ha=last_node.nodestack.ha, cliha=last_node.clientstack.ha) integrate_token_plugin_in_node(restarted_node) integrate_fees_plugin_in_node(restarted_node) tl = restarted_node.getLedger(TOKEN_LEDGER_ID) for node in nodeSetWithIntegratedTokenPlugin: token_ledger = node.getLedger(TOKEN_LEDGER_ID) assert token_ledger.size > tl.size looper.add(restarted_node) nodeSetWithIntegratedTokenPlugin.append(restarted_node) ensure_all_nodes_have_same_data(looper, nodeSetWithIntegratedTokenPlugin)
def new_node(node_name, tdir, node_ha, client_ha, tconf, plugin_path, nodeClass): config_helper = PNodeConfigHelper(node_name, tconf, chroot=tdir) node = nodeClass(node_name, config_helper=config_helper, config=tconf, ha=node_ha, cliha=client_ha, pluginPaths=plugin_path) return node
def testChangeNodeHaBack(looper, txnPoolNodeSet, tdir, tconf, steward1, stewardWallet, nodeThetaAdded): """ The case: The Node HA is updated with some HA (let's name it 'correct' HA). Then the Steward makes a mistake and sends the NODE txn with other HA ('wrong' HA). The Steward replaces back 'wrong' HA by 'correct' HA sending yet another one NODE txn. """ steward, stewardWallet, theta = nodeThetaAdded clientHa = theta.cliNodeReg['ThetaC'] # use the same client HA # do all exercises without the Node theta.stop() looper.removeProdable(name=theta.name) # step 1: set 'correct' HA correctNodeHa = genHa(1) op = { ALIAS: theta.name, NODE_IP: correctNodeHa.host, NODE_PORT: correctNodeHa.port, CLIENT_IP: clientHa.host, CLIENT_PORT: clientHa.port, } updateNodeData(looper, steward, stewardWallet, theta, op) # step 2: set 'wrong' HA wrongNodeHa = genHa(1) op.update({NODE_IP: wrongNodeHa.host, NODE_PORT: wrongNodeHa.port}) updateNodeData(looper, steward, stewardWallet, theta, op) # step 3: set 'correct' HA back op.update({NODE_IP: correctNodeHa.host, NODE_PORT: correctNodeHa.port}) updateNodeData(looper, steward, stewardWallet, theta, op) # In order to save the time the pool connection is not maintaining # during the steps, only the final result is checked. config_helper = PNodeConfigHelper(theta.name, tconf, chroot=tdir) restartedNode = TestNode(theta.name, ledger_dir=config_helper.ledger_dir, keys_dir=config_helper.keys_dir, genesis_dir=config_helper.genesis_dir, plugins_dir=config_helper.plugins_dir, config=tconf, ha=correctNodeHa, cliha=clientHa) looper.add(restartedNode) txnPoolNodeSet[-1] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet)) # check Theta HA for n in txnPoolNodeSet: assert n.nodeReg['Theta'] == correctNodeHa
def test_catchup_with_lost_first_consistency_proofs(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath, monkeypatch, lost_count): '''Skip processing of first lost_count CONSISTENCY_PROOFs in catchup. In this case catchup node has no quorum with f+1 CONSISTENCY_PROOFs for the longer transactions list. It need to request CONSISTENCY_PROOFs again and finishes catchup. Test makes sure that the node eventually finishes catchup''' node_to_disconnect = txnPoolNodeSet[-1] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) # restart node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) nodeHa, nodeCHa = HA(*node_to_disconnect.nodestack.ha), HA( *node_to_disconnect.clientstack.ha) config_helper = PNodeConfigHelper(node_to_disconnect.name, tconf, chroot=tdir) node_to_disconnect = TestNode(node_to_disconnect.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) def unpatch_after_call(proof, frm): global call_count call_count += 1 if call_count >= lost_count: # unpatch processConsistencyProof after lost_count calls node_to_disconnect.nodeMsgRouter.add((ConsistencyProof, node_to_disconnect.ledgerManager.processConsistencyProof)) call_count = 0 # patch processConsistencyProof node_to_disconnect.nodeMsgRouter.add((ConsistencyProof, unpatch_after_call)) # add node_to_disconnect to pool looper.add(node_to_disconnect) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc_backup'])
def initLocalKeys(tdir_for_func, tconf_for_func, nodeReg): for nName in nodeReg.keys(): sigseed = randomString(32).encode() config_helper = PNodeConfigHelper(nName, tconf_for_func, chroot=tdir_for_func) initNodeKeysForBothStacks(nName, config_helper.keys_dir, sigseed, override=True) logger.debug('Created keys for {}'.format(nName))
def test_primary_selection_after_primary_demotion_and_pool_restart( looper, txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes, tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. """ logger.info( "1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] client, wallet = stewardAndWalletForMasterNode node_data = {ALIAS: master_node.name, SERVICES: []} updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [ node for node in txnPoolNodeSet if node.name != master_node.name ] ensureElectionsDone(looper, restNodes) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("2. restart pool") # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) restNodes = [ node for node in restartedNodes if node.name != master_node.name ] looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name
def test_node(tdirWithPoolTxns, tdirWithDomainTxns, poolTxnNodeNames, tdirWithNodeKeepInited, tdir, tconf, allPluginsPath): node_name = poolTxnNodeNames[0] config_helper = PNodeConfigHelper(node_name, tconf, chroot=tdir) node = TestNode(node_name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath) node.view_changer = FakeSomething(view_change_in_progress=False, view_no=0) yield node node.onStopping()
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) looper = looper_without_nodeset_for_func initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) nodes = [] names = list(nodeReg.keys()) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for name in names: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) for i, node in enumerate(nodes): looper.add(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes)) logger.debug("connects") logger.debug("node order: {}".format(names)) logger.debug("waits: {}".format(waits)) stopNodes(nodes, looper) # # Giving some time for sockets to close, use eventually # time.sleep(1) for i, n in enumerate(nodes): n.start(looper.loop) looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper) logger.debug("reconnects") logger.debug("node order: {}".format(names)) logger.debug("rwaits: {}".format(rwaits))
def test_node(tdirWithPoolTxns, tdirWithDomainTxns, poolTxnNodeNames, tdirWithNodeKeepInited, tdir, tconf, allPluginsPath): node_name = poolTxnNodeNames[0] config_helper = PNodeConfigHelper(node_name, tconf, chroot=tdir) node = TestNode(node_name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath) yield node node.onStopping( ) # TODO stop won't call onStopping as we are in Stopped state
def test_catchup_with_lost_ledger_status(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath, monkeypatch, lost_count): '''Skip processing of lost_count Message Responses with LEDGER STATUS in catchup; test makes sure that the node eventually finishes catchup''' node_to_disconnect = txnPoolNodeSet[-1] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) # restart node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) nodeHa, nodeCHa = HA(*node_to_disconnect.nodestack.ha), HA( *node_to_disconnect.clientstack.ha) config_helper = PNodeConfigHelper(node_to_disconnect.name, tconf, chroot=tdir) node_to_disconnect = TestNode(node_to_disconnect.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) def unpatch_after_call(status, frm): global call_count call_count += 1 if call_count >= lost_count: # unpatch processLedgerStatus after lost_count calls node_to_disconnect.nodeMsgRouter.add((LedgerStatus, node_to_disconnect.ledgerManager.processLedgerStatus)) call_count = 0 # patch processLedgerStatus node_to_disconnect.nodeMsgRouter.add((LedgerStatus, unpatch_after_call)) # add node_to_disconnect to pool looper.add(node_to_disconnect) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc_backup'])
def testNodeCatchupFPlusOne(txnPoolNodeSet, poolAfterSomeTxns, tconf, tdir, tdirWithPoolTxns, allPluginsPath, testNodeClass): """ Check that f+1 nodes is enough for catchup """ looper, client, wallet = poolAfterSomeTxns assert len(txnPoolNodeSet) == 4 node1 = txnPoolNodeSet[-1] node0 = txnPoolNodeSet[-2] logger.debug("Stopping node0 with pool ledger size {}".format( node0.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node0, stopNode=True) looper.removeProdable(node0) logger.debug("Sending requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) logger.debug("Stopping node1 with pool ledger size {}".format( node1.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node1, stopNode=True) looper.removeProdable(node1) # Make sure new node got out of sync waitNodeDataInequality(looper, node0, *txnPoolNodeSet[:-2]) # TODO: Check if the node has really stopped processing requests? logger.debug("Starting the stopped node0") nodeHa, nodeCHa = HA(*node0.nodestack.ha), HA(*node0.clientstack.ha) config_helper = PNodeConfigHelper(node0.name, tconf, chroot=tdir) node0 = testNodeClass(node0.name, config_helper=config_helper, ha=nodeHa, cliha=nodeCHa, config=tconf, pluginPaths=allPluginsPath) looper.add(node0) logger.debug("Waiting for the node0 to catch up") waitNodeDataEquality(looper, node0, *txnPoolNodeSet[:-2]) logger.debug("Sending more requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 2) checkNodeDataForEquality(node0, *txnPoolNodeSet[:-2])
def addNodeBack(node_set, looper: Looper, node: Node, tconf, tdir) -> TestNode: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) for node in node_set: if node.name != restartedNode.name: node.nodestack.reconnectRemoteWithName(restartedNode.name) node_set.append(restartedNode) looper.add(restartedNode) return restartedNode
def start_newly_added_node(looper, node_name, tdir, sigseed, node_ha, client_ha, tconf, auto_start, plugin_path, nodeClass): config_helper = PNodeConfigHelper(node_name, tconf, chroot=tdir) node = nodeClass(node_name, config_helper=config_helper, config=tconf, ha=node_ha, cliha=client_ha, pluginPaths=plugin_path) if auto_start: looper.add(node) return node