def test_genNodeReg(self): l = 10 nr = genNodeReg(count=l) cli = nr.extractCliNodeReg() # lengths are correct assert len(cli) == l assert len(nr) == l # node names don't overlap assert set(nr.keys()) & set(cli.keys()) == set() # checking port values nrports = [x.ha.port for x in nr.values()] cliports = [x.port for x in cli.values()] # ports are all numbers allports = set() for s in [nrports, cliports]: for p in s: assert type(p) == int allports.add(p) # ports don't overlap assert len(allports) == l*2
def test_genNodeReg(self): l = 10 nr = genNodeReg(count=l) cli = nr.extractCliNodeReg() # lengths are correct assert len(cli) == l assert len(nr) == l # node names don't overlap assert set(nr.keys()) & set(cli.keys()) == set() # checking port values nrports = [x.ha.port for x in nr.values()] cliports = [x.port for x in cli.values()] # ports are all numbers allports = set() for s in [nrports, cliports]: for p in s: assert type(p) == int allports.add(p) # ports don't overlap assert len(allports) == l * 2
def testNode(pluginManager, tdir): name = randomText(20) nodeReg = genNodeReg(names=[name]) ha, cliname, cliha = nodeReg[name] return TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(nodeReg), basedirpath=tdir, primaryDecider=None, pluginPaths=None, seed=randomSeed())
def testSystemStats(monkeypatch, tdir): """ Checking if monitor captures system performance data """ cpu = 10 ram = 15 bytes = 1024 class TestRam(): def __init__(self): nonlocal ram self.percent = ram class TestBytes(): def __init__(self): nonlocal bytes self.bytes_sent = bytes self.bytes_recv = 0 def test_cpu_percent(interval): nonlocal cpu return cpu def test_virtual_memory(): return TestRam() def test_traffic(): return TestBytes() name = 'TestNodeMonitorPerfStats' nodeReg = genNodeReg(names=[name]) ha, cliname, cliha = nodeReg[name] node = TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(nodeReg), basedirpath=tdir, primaryDecider=None, pluginPaths=None) monkeypatch.setattr(psutil, 'cpu_percent', test_cpu_percent) monkeypatch.setattr(psutil, 'virtual_memory', test_virtual_memory) monkeypatch.setattr(psutil, 'net_io_counters', test_traffic) node.monitor.lastKnownTraffic = 0 data1 = node.monitor.captureSystemPerformance() assert 'cpu' in data1 assert 'ram' in data1 assert 'traffic' in data1 assert data1['cpu']['value'] == cpu assert data1['ram']['value'] == ram assert data1['traffic']['value'] == bytes / 1024 cpu = 50 ram = 60 bytes = 2048 assert node.monitor.lastKnownTraffic == data1['traffic']['value'] data2 = node.monitor.captureSystemPerformance() assert data2['cpu']['value'] == cpu assert data2['ram']['value'] == ram assert data2['traffic']['value'] == bytes / 1024 - data1['traffic']['value']
def testNode(pluginManager, tdir, tconf, node_config_helper_class): name = randomText(20) nodeReg = genNodeReg(names=[name]) ha, cliname, cliha = nodeReg[name] config_helper = node_config_helper_class(name, tconf, chroot=tdir) node = TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(nodeReg), config_helper=config_helper, config=tconf, primaryDecider=None, pluginPaths=None, seed=randomSeed()) node.start(None) yield node node.stop()
def testKeyShareParty(tdir_for_func): """ connections to all nodes should be successfully established when key sharing is enabled. """ nodeReg = genNodeReg(5) logger.debug("-----sharing keys-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: looper.run(checkNodesConnected(nodeSet)) logger.debug("-----key sharing done, connect after key sharing-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as loop: loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logger.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run( eventually(checkRequestReturnedToNode, node, wallet1.defaultId, req.reqId, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testNode(pluginManager, tdir): name = randomText(20) nodeReg = genNodeReg(names=[name]) ha, cliname, cliha = nodeReg[name] node = TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(nodeReg), basedirpath=tdir, base_data_dir=tdir, primaryDecider=None, pluginPaths=None, seed=randomSeed()) node.start(None) yield node node.stop()
def testKeyShareParty(tdir_for_func): """ connections to all nodes should be successfully established when key sharing is enabled. """ nodeReg = genNodeReg(5) logger.debug("-----sharing keys-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() looper.run(checkNodesConnected(nodeSet)) logger.debug("-----key sharing done, connect after key sharing-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as loop: loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logger.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run(eventually(checkRequestReturnedToNode, node, wallet1.defaultId, req.reqId, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def nodeReg(request) -> Dict[str, HA]: nodeCount = getValueFromModule(request, "nodeCount", 4) return genNodeReg(count=nodeCount)
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack( nodeSet, looper, nodeNames[i]) for i in range( 0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually( lambda: assertLength( getPendingRequestsForReplica( nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay