def test_primary_selection_after_demoted_primary_node_promotion( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes): """ Demote primary of master instance, wait for view change and promote it back. Check primaries for instances. """ assert len(txnPoolNodeSet) == 4 # Check primaries after test setup. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 0 assert primariesIdxs[1] == 1 master_node = txnPoolMasterNodes[0] # Demote primary of master instance. node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [node for node in txnPoolNodeSet if node.name != master_node.name] ensureElectionsDone(looper, restNodes) # Check that there is only one instance now, check it's primary. primariesIdxs = getPrimaryNodesIdxs(restNodes) assert len(primariesIdxs) == 1 assert primariesIdxs[0] == 1 # Ensure pool is working properly. sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) # Promote demoted node back. sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[VALIDATOR]) ensureElectionsDone(looper, txnPoolNodeSet) # Ensure pool is working properly. sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) # Check that there are two instances again, check their primaries. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 2 assert primariesIdxs[1] == 3
def test_primary_selection_after_demoted_primary_node_promotion( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes): """ Demote primary of master instance, wait for view change and promote it back. Check primaries for instances. """ assert len(txnPoolNodeSet) == 4 # Check primaries after test setup. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 0 assert primariesIdxs[1] == 1 master_node = txnPoolMasterNodes[0] # Demote primary of master instance. node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [node for node in txnPoolNodeSet if node.name != master_node.name] ensureElectionsDone(looper, restNodes) # Check that there is only one instance now, check it's primary. primariesIdxs = getPrimaryNodesIdxs(restNodes) assert len(primariesIdxs) == 1 assert primariesIdxs[0] == 1 # Ensure pool is working properly. sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) # Promote demoted node back. sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[VALIDATOR]) # Ensure pool is working properly. sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) # Check that there are two instances again, check their primaries. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 2 assert primariesIdxs[1] == 3
def test_primary_selection_after_demoted_primary_node_promotion( looper, txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes): """ Demote primary of master instance, wait for view change and promote it back. Check primaries for instances. """ assert len(txnPoolNodeSet) == 4 # Check primaries after test setup. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 0 assert primariesIdxs[1] == 1 master_node = txnPoolMasterNodes[0] client, wallet = stewardAndWalletForMasterNode # Demote primary of master instance. node_data = {ALIAS: master_node.name, SERVICES: []} updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [ node for node in txnPoolNodeSet if node.name != master_node.name ] ensureElectionsDone(looper, restNodes) # Check that there is only one instance now, check it's primary. primariesIdxs = getPrimaryNodesIdxs(restNodes) assert len(primariesIdxs) == 1 assert primariesIdxs[0] == 1 # Ensure pool is working properly. sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) # Promote demoted node back. node_data = {ALIAS: master_node.name, SERVICES: [VALIDATOR]} updateNodeData(looper, client, wallet, master_node, node_data) # Ensure pool is working properly. sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) # Check that there are two instances again, check their primaries. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) assert len(primariesIdxs) == 2 assert primariesIdxs[0] == 2 assert primariesIdxs[1] == 3
def test_primary_selection_after_primary_demotion_and_pool_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes, tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. """ logger.info("1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [node for node in txnPoolNodeSet if node.name != master_node.name] ensureElectionsDone(looper, restNodes) # ensure pool is working properly logger.info("2. restart pool") # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) restNodes = [node for node in restartedNodes if node.name != master_node.name] looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name
def test_primary_selection_after_primary_demotion_and_pool_restart( looper, txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes, tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. """ logger.info( "1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] client, wallet = stewardAndWalletForMasterNode node_data = {ALIAS: master_node.name, SERVICES: []} updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [ node for node in txnPoolNodeSet if node.name != master_node.name ] ensureElectionsDone(looper, restNodes) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("2. restart pool") # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) restNodes = [ node for node in restartedNodes if node.name != master_node.name ] looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames, tdirWithClientPoolTxns, txnPoolNodeSet, txnPoolMasterNodes): primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) master_node = txnPoolMasterNodes[0] stewardName = poolTxnStewardNames[primariesIdxs[0]] stewardsSeed = poolTxnData["seeds"][stewardName].encode() stewardClient, stewardWallet = buildPoolClientAndWallet( (stewardName, stewardsSeed), tdirWithClientPoolTxns) looper.add(stewardClient) looper.run(stewardClient.ensureConnectedToNodes()) return stewardClient, stewardWallet
def test_primary_selection_after_primary_demotion_and_pool_restart( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes, tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. """ logger.info( "1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [ node for node in txnPoolNodeSet if node.name != master_node.name ] ensureElectionsDone(looper, restNodes) # ensure pool is working properly logger.info("2. restart pool") # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) restNodes = [ node for node in restartedNodes if node.name != master_node.name ] looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name
def txnPoolMasterNodes(txnPoolNodeSet): primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) return txnPoolNodeSet[primariesIdxs[0]], txnPoolNodeSet[primariesIdxs[1]]
def txnPoolMasterNodes(txnPoolNodeSet): primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) return txnPoolNodeSet[primariesIdxs[0]], txnPoolNodeSet[primariesIdxs[1]]