def simulate_slow_master(looper, nodeSet, wallet,
client, delay=10, num_reqs=4):
m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, delay)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, num_reqs)
return m_primary_node
def simulate_slow_master(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, delay=10, num_reqs=4):
m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, delay)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, num_reqs)
return m_primary_node
def viewChangeDone(nodeSet, looper, up, wallet1, client1, viewNo):
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo+1),
retryWait=1, timeout=20))
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
pr = getPrimaryReplica(txnPoolNodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
for n in txnPoolNodeSet:
sentInstChanges[n.name] = node_sent_instance_changes_count(n)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
backup_replica = txnPoolNodeSet[0].replicas[1]
backup_last_ordered_before = backup_replica.last_ordered_3pc
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 4)
# make sure that backups also ordered at least 1 batch to be able to track performance degradation
looper.run(
eventually(lambda: assertExp(backup_replica.last_ordered_3pc >
backup_last_ordered_before)))
for n in txnPoolNodeSet:
n.checkPerformance()
# Check that view change happened for all nodes
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1)
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in txnPoolNodeSet:
if n.name != relucatantNode.name:
assert node_sent_instance_changes_count(n) > sentInstChanges.get(
n.name, 0)
else:
assert node_sent_instance_changes_count(n) == sentInstChanges.get(
n.name, 0)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def viewChangeDone(nodeSet, looper, up, wallet1, client1, viewNo):
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
looper.run(
eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
retryWait=1,
timeout=20))
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet,
viewNo + 1), retryWait=1, timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client,
wallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(eventually(checkNodeLedgersForEquality, nodeTheta,
*txnPoolNodeSet[:-1], retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(
0,
viewNo,
10,
wallet.defaultId,
wallet._getIdData().lastReqId+1,
"random digest",
time.time()
)
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet,
viewNo + 1), retryWait=1, timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client,
wallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(eventually(checkNodeLedgersForEquality, nodeTheta,
*txnPoolNodeSet[:-1], retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(
0,
viewNo,
10,
wallet.defaultId,
wallet._getIdData().lastReqId+1,
"random digest",
time.time()
)
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
pr = getPrimaryReplica(txnPoolNodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = ViewChanger.sendInstanceChange.__name__
for n in txnPoolNodeSet:
sentInstChanges[n.name] = n.view_changer.spylog.count(
instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 4)
for n in txnPoolNodeSet:
n.checkPerformance()
# Check that view change happened for all nodes
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1)
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in txnPoolNodeSet:
if n.name != relucatantNode.name:
assert n.view_changer.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.view_changer.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward,
viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
pr = getPrimaryReplica(txnPoolNodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = ViewChanger.sendInstanceChange.__name__
for n in txnPoolNodeSet:
sentInstChanges[n.name] = n.view_changer.spylog.count(instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 4)
# Check that view change happened for all nodes
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1)
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in txnPoolNodeSet:
if n.name != relucatantNode.name:
assert n.view_changer.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.view_changer.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
pr = getPrimaryReplica(nodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = Node.sendInstanceChange.__name__
for n in nodeSet:
sentInstChanges[n.name] = n.spylog.count(instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
# Check that view change happened for all nodes
looper.run(
eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
retryWait=1,
timeout=20))
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in nodeSet:
if n.name != relucatantNode.name:
assert n.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
pr = getPrimaryReplica(nodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = Node.sendInstanceChange.__name__
for n in nodeSet:
sentInstChanges[n.name] = n.spylog.count(instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
# Check that view change happened for all nodes
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
retryWait=1, timeout=20))
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in nodeSet:
if n.name != relucatantNode.name:
assert n.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
