def simulate_slow_master(looper, nodeSet, wallet, client, delay=10, num_reqs=4): m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, delay) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, num_reqs) return m_primary_node
def simulate_slow_master(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, delay=10, num_reqs=4): m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, delay) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, num_reqs) return m_primary_node
def viewChangeDone(nodeSet, looper, up, wallet1, client1, viewNo): # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo+1), retryWait=1, timeout=20))
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, 10) pr = getPrimaryReplica(txnPoolNodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} for n in txnPoolNodeSet: sentInstChanges[n.name] = node_sent_instance_changes_count(n) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) backup_replica = txnPoolNodeSet[0].replicas[1] backup_last_ordered_before = backup_replica.last_ordered_3pc sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 4) # make sure that backups also ordered at least 1 batch to be able to track performance degradation looper.run( eventually(lambda: assertExp(backup_replica.last_ordered_3pc > backup_last_ordered_before))) for n in txnPoolNodeSet: n.checkPerformance() # Check that view change happened for all nodes waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in txnPoolNodeSet: if n.name != relucatantNode.name: assert node_sent_instance_changes_count(n) > sentInstChanges.get( n.name, 0) else: assert node_sent_instance_changes_count(n) == sentInstChanges.get( n.name, 0) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert m_primary_node.name != new_m_primary_node.name ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def viewChangeDone(nodeSet, looper, up, wallet1, client1, viewNo): # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) looper.run( eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1), retryWait=1, timeout=20))
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, newNodeCaughtUp, tdirWithPoolTxns, tconf, allPluginsPath): """ Node discards 3-phase and election messages from view nos that it does not know of (view nos before it joined the pool) :return: """ looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns viewNo = nodeX.viewNo # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, 10) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4) looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet, viewNo + 1), retryWait=1, timeout=20)) newStewardName = "testClientSteward" + randomString(3) nodeName = "Theta" _, _, nodeTheta = addNewStewardAndNode(looper, client, wallet, newStewardName, nodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(client.ensureConnectedToNodes()) looper.run(eventually(checkNodeLedgersForEquality, nodeTheta, *txnPoolNodeSet[:-1], retryWait=1, timeout=5)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=10) electMsg = Nomination(nodeX.name, 0, viewNo) threePMsg = PrePrepare( 0, viewNo, 10, wallet.defaultId, wallet._getIdData().lastReqId+1, "random digest", time.time() ) ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid nodeX.send(electMsg, ridTheta) nodeX.send(threePMsg, ridTheta) nodeX.send(electMsg, ridTheta) looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg, 'un-acceptable viewNo', retryWait=1, timeout=5)) nodeX.send(threePMsg, ridTheta) looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg, 'un-acceptable viewNo', retryWait=1, timeout=5))
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, 10) pr = getPrimaryReplica(txnPoolNodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = ViewChanger.sendInstanceChange.__name__ for n in txnPoolNodeSet: sentInstChanges[n.name] = n.view_changer.spylog.count( instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 4) for n in txnPoolNodeSet: n.checkPerformance() # Check that view change happened for all nodes waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in txnPoolNodeSet: if n.name != relucatantNode.name: assert n.view_changer.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.view_changer.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert m_primary_node.name != new_m_primary_node.name ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, 10) pr = getPrimaryReplica(txnPoolNodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = ViewChanger.sendInstanceChange.__name__ for n in txnPoolNodeSet: sentInstChanges[n.name] = n.view_changer.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 4) # Check that view change happened for all nodes waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in txnPoolNodeSet: if n.name != relucatantNode.name: assert n.view_changer.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.view_changer.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert m_primary_node.name != new_m_primary_node.name ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) pr = getPrimaryReplica(nodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = Node.sendInstanceChange.__name__ for n in nodeSet: sentInstChanges[n.name] = n.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Check that view change happened for all nodes looper.run( eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1), retryWait=1, timeout=20)) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in nodeSet: if n.name != relucatantNode.name: assert n.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0)
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) pr = getPrimaryReplica(nodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = Node.sendInstanceChange.__name__ for n in nodeSet: sentInstChanges[n.name] = n.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Check that view change happened for all nodes looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1), retryWait=1, timeout=20)) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in nodeSet: if n.name != relucatantNode.name: assert n.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0)