def test_preprepares_and_prepares_recovery_after_catchup( tdir, tconf, looper, testNodeClass, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, allPluginsPath, chkFreqPatched): """ Test that all preprepares and prepares are recovered from the audit ledger after reboot. """ node_to_restart = txnPoolNodeSet[-1] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, NUM_OF_REQ) # Check that all of the nodes except the slows one ordered the request looper.run(eventually(check_last_ordered, txnPoolNodeSet, (0, NUM_OF_REQ))) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_restart, timeout=len(txnPoolNodeSet), stopNode=True) looper.removeProdable(node_to_restart) txnPoolNodeSet.remove(node_to_restart) restarted_node = start_stopped_node(node_to_restart, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(restarted_node) looper.runFor(waits.expectedNodeStartUpTimeout()) looper.run(checkNodesConnected(txnPoolNodeSet)) check_prepared(txnPoolNodeSet)
def testMsgSendingTime(pool): nodes = pool.nodeset msg = randomMsg() timeout = waits.expectedNodeStartUpTimeout() pool.looper.run( sendMessageAndCheckDelivery(nodes[0], nodes[1], msg, customTimeout=timeout))
def testMsgSendingTime(pool, nodeReg): nodeNames = list(nodeReg.keys()) msg = randomMsg() timeout = waits.expectedNodeStartUpTimeout() pool.looper.run( sendMessageAndCheckDelivery(pool.nodeset, nodeNames[0], nodeNames[1], msg, customTimeout=timeout))
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): logger.info("Add back the {} node and see status of {}". format(ordinal(nodeIdx + 1), expectedStatus)) addNodeBack(nodeSet, looper, nodeNames[nodeIdx]) timeout = waits.expectedNodeStartUpTimeout() + \ waits.expectedPoolInterconnectionTime(len(nodeSet)) looper.run(eventually(checkNodeStatusRemotesAndF, expectedStatus, nodeIdx, retryWait=1, timeout=timeout))
def testSelfNominationDelay(tdir_for_func): nodeNames = ["testA", "testB", "testC", "testD"] with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) delay = 30 # Add node A nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) nodeA.delaySelfNomination(delay) nodesBCD = [] for name in nodeNames[1:]: # nodesBCD.append(nodeSet.addNode(name, i+1, AutoMode.never)) nodesBCD.append(addNodeBack(nodeSet, looper, name)) # Ensuring that NodeA is started before any other node to demonstrate # that it is delaying self nomination timeout = waits.expectedNodeStartUpTimeout() looper.run( eventually(lambda: assertExp(nodeA.isReady()), retryWait=1, timeout=timeout)) ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1) # node A should not have any primary replica timeout = waits.expectedNodeStartUpTimeout() looper.run( eventually(lambda: assertExp(not nodeA.hasPrimary), retryWait=1, timeout=timeout)) # Make sure that after at the most 30 seconds, nodeA's # `startElection` is called looper.run( eventually(lambda: assertExp( len(nodeA.spylog.getAll(Node.decidePrimaries.__name__)) > 0 ), retryWait=1, timeout=delay))
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): logger.info("Add back the {} node and see status of {}". format(ordinal(nodeIdx + 1), expectedStatus)) addNodeBack( current_node_set, looper, get_node_by_name(txnPoolNodeSet, nodeNames[nodeIdx]), tconf, tdir) looper.run(checkNodesConnected(current_node_set)) timeout = waits.expectedNodeStartUpTimeout() + \ waits.expectedPoolInterconnectionTime(len(current_node_set)) # TODO: Probably it's better to modify waits.* functions timeout *= 1.5 looper.run(eventually(checkNodeStatusRemotesAndF, expectedStatus, nodeIdx, retryWait=1, timeout=timeout))
def waitNodeStarted(cli, nodeName): # Node name should be in cli.nodes assert nodeName in cli.nodes def chk(): msgs = {stmt['msg'] for stmt in cli.printeds} print("checking for {}".format(nodeName)) print(msgs) assert "{} added replica {}:0 to instance 0 (master)" \ .format(nodeName, nodeName) in msgs assert "{} added replica {}:1 to instance 1 (backup)" \ .format(nodeName, nodeName) in msgs assert "{} listening for other nodes at {}:{}" \ .format(nodeName, *cli.nodes[nodeName].nodestack.ha) in msgs startUpTimeout = waits.expectedNodeStartUpTimeout() cli.looper.run(eventually(chk, timeout=startUpTimeout))
def do_view_change_with_delayed_commits_and_node_restarts( fast_nodes, slow_nodes, nodes_to_restart, old_view_no, old_last_ordered, looper, sdk_pool_handle, sdk_wallet_client, tconf, tdir, all_plugins_path, wait_for_catchup=False): """ Delays commits without processing on `slow_nodes`, restarts `nodes_to_restart`, triggers view change, and confirms that view changed completed successfully and that the ledgers are consistent and in sync. :param fast_nodes: Nodes that will order the requests :param slow_nodes: Nodes whose commits will be delay, and that will not order the requests :param nodes_to_restart: Nodes that will be restarted :param old_view_no: View that we started from :param old_last_ordered: Last ordered 3pc txn before we did any requests :param wait_for_catchup: Should we wait for restarted nodes to finish catchup """ nodes = fast_nodes + slow_nodes slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes] # Delay commits on `slow_nodes` with delay_rules_without_processing(slow_stashers, cDelay()): request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) # Check that all of the nodes except the slows one ordered the request looper.run( eventually(check_last_ordered, fast_nodes, (old_view_no, old_last_ordered[1] + 1))) looper.run(eventually(check_last_ordered, slow_nodes, old_last_ordered)) # Restart nodes for node in nodes_to_restart: disconnect_node_and_ensure_disconnected(looper, nodes, node, timeout=len(nodes_to_restart), stopNode=True) looper.removeProdable(node) nodes.remove(node) restarted_node = start_stopped_node(node, looper, tconf, tdir, all_plugins_path) nodes.append(restarted_node) looper.runFor(waits.expectedNodeStartUpTimeout()) looper.run(checkNodesConnected(nodes)) if wait_for_catchup: ensure_all_nodes_have_same_data(looper, nodes) # Trigger view change on all nodes for node in nodes: node.view_changer.on_master_degradation() assert len(nodes) == len(slow_nodes) + len(fast_nodes) # Assert that view change was successful and that ledger data is consistent waitForViewChange(looper, nodes, expectedViewNo=(old_view_no + 1), customTimeout=waits.expectedPoolViewChangeStartedTimeout( len(nodes))) ensureElectionsDone(looper=looper, nodes=nodes) ensure_all_nodes_have_same_data(looper, nodes) sdk_get_reply(looper, request) sdk_ensure_pool_functional(looper, nodes, sdk_wallet_client, sdk_pool_handle)