def test_preprepares_and_prepares_recovery_after_catchup(
tdir, tconf, looper, testNodeClass, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, allPluginsPath, chkFreqPatched):
"""
Test that all preprepares and prepares are recovered from the audit ledger after reboot.
"""
node_to_restart = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, NUM_OF_REQ)
# Check that all of the nodes except the slows one ordered the request
looper.run(eventually(check_last_ordered, txnPoolNodeSet, (0, NUM_OF_REQ)))
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node_to_restart,
timeout=len(txnPoolNodeSet),
stopNode=True)
looper.removeProdable(node_to_restart)
txnPoolNodeSet.remove(node_to_restart)
restarted_node = start_stopped_node(node_to_restart, looper, tconf, tdir,
allPluginsPath)
txnPoolNodeSet.append(restarted_node)
looper.runFor(waits.expectedNodeStartUpTimeout())
looper.run(checkNodesConnected(txnPoolNodeSet))
check_prepared(txnPoolNodeSet)
def testMsgSendingTime(pool):
nodes = pool.nodeset
msg = randomMsg()
timeout = waits.expectedNodeStartUpTimeout()
pool.looper.run(
sendMessageAndCheckDelivery(nodes[0],
nodes[1],
msg,
customTimeout=timeout))
def testMsgSendingTime(pool, nodeReg):
nodeNames = list(nodeReg.keys())
msg = randomMsg()
timeout = waits.expectedNodeStartUpTimeout()
pool.looper.run(
sendMessageAndCheckDelivery(pool.nodeset,
nodeNames[0],
nodeNames[1],
msg,
customTimeout=timeout))
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logger.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(nodeSet, looper, nodeNames[nodeIdx])
timeout = waits.expectedNodeStartUpTimeout() + \
waits.expectedPoolInterconnectionTime(len(nodeSet))
looper.run(eventually(checkNodeStatusRemotesAndF,
expectedStatus,
nodeIdx,
retryWait=1, timeout=timeout))
def testSelfNominationDelay(tdir_for_func):
nodeNames = ["testA", "testB", "testC", "testD"]
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
delay = 30
# Add node A
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
nodeA.delaySelfNomination(delay)
nodesBCD = []
for name in nodeNames[1:]:
# nodesBCD.append(nodeSet.addNode(name, i+1, AutoMode.never))
nodesBCD.append(addNodeBack(nodeSet, looper, name))
# Ensuring that NodeA is started before any other node to demonstrate
# that it is delaying self nomination
timeout = waits.expectedNodeStartUpTimeout()
looper.run(
eventually(lambda: assertExp(nodeA.isReady()),
retryWait=1,
timeout=timeout))
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1)
# node A should not have any primary replica
timeout = waits.expectedNodeStartUpTimeout()
looper.run(
eventually(lambda: assertExp(not nodeA.hasPrimary),
retryWait=1,
timeout=timeout))
# Make sure that after at the most 30 seconds, nodeA's
# `startElection` is called
looper.run(
eventually(lambda: assertExp(
len(nodeA.spylog.getAll(Node.decidePrimaries.__name__)) > 0
),
retryWait=1,
timeout=delay))
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logger.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(
current_node_set, looper,
get_node_by_name(txnPoolNodeSet, nodeNames[nodeIdx]),
tconf, tdir)
looper.run(checkNodesConnected(current_node_set))
timeout = waits.expectedNodeStartUpTimeout() + \
waits.expectedPoolInterconnectionTime(len(current_node_set))
# TODO: Probably it's better to modify waits.* functions
timeout *= 1.5
looper.run(eventually(checkNodeStatusRemotesAndF,
expectedStatus,
nodeIdx,
retryWait=1, timeout=timeout))
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logger.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(
current_node_set, looper,
get_node_by_name(txnPoolNodeSet, nodeNames[nodeIdx]),
tconf, tdir)
looper.run(checkNodesConnected(current_node_set))
timeout = waits.expectedNodeStartUpTimeout() + \
waits.expectedPoolInterconnectionTime(len(current_node_set))
# TODO: Probably it's better to modify waits.* functions
timeout *= 1.5
looper.run(eventually(checkNodeStatusRemotesAndF,
expectedStatus,
nodeIdx,
retryWait=1, timeout=timeout))
def waitNodeStarted(cli, nodeName):
# Node name should be in cli.nodes
assert nodeName in cli.nodes
def chk():
msgs = {stmt['msg'] for stmt in cli.printeds}
print("checking for {}".format(nodeName))
print(msgs)
assert "{} added replica {}:0 to instance 0 (master)" \
.format(nodeName, nodeName) in msgs
assert "{} added replica {}:1 to instance 1 (backup)" \
.format(nodeName, nodeName) in msgs
assert "{} listening for other nodes at {}:{}" \
.format(nodeName, *cli.nodes[nodeName].nodestack.ha) in msgs
startUpTimeout = waits.expectedNodeStartUpTimeout()
cli.looper.run(eventually(chk, timeout=startUpTimeout))
def do_view_change_with_delayed_commits_and_node_restarts(
fast_nodes,
slow_nodes,
nodes_to_restart,
old_view_no,
old_last_ordered,
looper,
sdk_pool_handle,
sdk_wallet_client,
tconf,
tdir,
all_plugins_path,
wait_for_catchup=False):
"""
Delays commits without processing on `slow_nodes`, restarts `nodes_to_restart`, triggers view change, and confirms
that view changed completed successfully and that the ledgers are consistent and in sync.
:param fast_nodes: Nodes that will order the requests
:param slow_nodes: Nodes whose commits will be delay, and that will not order the requests
:param nodes_to_restart: Nodes that will be restarted
:param old_view_no: View that we started from
:param old_last_ordered: Last ordered 3pc txn before we did any requests
:param wait_for_catchup: Should we wait for restarted nodes to finish catchup
"""
nodes = fast_nodes + slow_nodes
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
# Delay commits on `slow_nodes`
with delay_rules_without_processing(slow_stashers, cDelay()):
request = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_client)
# Check that all of the nodes except the slows one ordered the request
looper.run(
eventually(check_last_ordered, fast_nodes,
(old_view_no, old_last_ordered[1] + 1)))
looper.run(eventually(check_last_ordered, slow_nodes,
old_last_ordered))
# Restart nodes
for node in nodes_to_restart:
disconnect_node_and_ensure_disconnected(looper,
nodes,
node,
timeout=len(nodes_to_restart),
stopNode=True)
looper.removeProdable(node)
nodes.remove(node)
restarted_node = start_stopped_node(node, looper, tconf, tdir,
all_plugins_path)
nodes.append(restarted_node)
looper.runFor(waits.expectedNodeStartUpTimeout())
looper.run(checkNodesConnected(nodes))
if wait_for_catchup:
ensure_all_nodes_have_same_data(looper, nodes)
# Trigger view change on all nodes
for node in nodes:
node.view_changer.on_master_degradation()
assert len(nodes) == len(slow_nodes) + len(fast_nodes)
# Assert that view change was successful and that ledger data is consistent
waitForViewChange(looper,
nodes,
expectedViewNo=(old_view_no + 1),
customTimeout=waits.expectedPoolViewChangeStartedTimeout(
len(nodes)))
ensureElectionsDone(looper=looper, nodes=nodes)
ensure_all_nodes_have_same_data(looper, nodes)
sdk_get_reply(looper, request)
sdk_ensure_pool_functional(looper, nodes, sdk_wallet_client,
sdk_pool_handle)
