def test_genNodeReg(self):
l = 10
nr = genNodeReg(count=l)
cli = nr.extractCliNodeReg()
# lengths are correct
assert len(cli) == l
assert len(nr) == l
# node names don't overlap
assert set(nr.keys()) & set(cli.keys()) == set()
# checking port values
nrports = [x.ha.port for x in nr.values()]
cliports = [x.port for x in cli.values()]
# ports are all numbers
allports = set()
for s in [nrports, cliports]:
for p in s:
assert type(p) == int
allports.add(p)
# ports don't overlap
assert len(allports) == l*2
def test_genNodeReg(self):
l = 10
nr = genNodeReg(count=l)
cli = nr.extractCliNodeReg()
# lengths are correct
assert len(cli) == l
assert len(nr) == l
# node names don't overlap
assert set(nr.keys()) & set(cli.keys()) == set()
# checking port values
nrports = [x.ha.port for x in nr.values()]
cliports = [x.port for x in cli.values()]
# ports are all numbers
allports = set()
for s in [nrports, cliports]:
for p in s:
assert type(p) == int
allports.add(p)
# ports don't overlap
assert len(allports) == l * 2
def testNode(pluginManager, tdir):
name = randomText(20)
nodeReg = genNodeReg(names=[name])
ha, cliname, cliha = nodeReg[name]
return TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha,
nodeRegistry=copy(nodeReg), basedirpath=tdir,
primaryDecider=None, pluginPaths=None, seed=randomSeed())
def testSystemStats(monkeypatch, tdir):
"""
Checking if monitor captures system performance data
"""
cpu = 10
ram = 15
bytes = 1024
class TestRam():
def __init__(self):
nonlocal ram
self.percent = ram
class TestBytes():
def __init__(self):
nonlocal bytes
self.bytes_sent = bytes
self.bytes_recv = 0
def test_cpu_percent(interval):
nonlocal cpu
return cpu
def test_virtual_memory():
return TestRam()
def test_traffic():
return TestBytes()
name = 'TestNodeMonitorPerfStats'
nodeReg = genNodeReg(names=[name])
ha, cliname, cliha = nodeReg[name]
node = TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(nodeReg), basedirpath=tdir,
primaryDecider=None, pluginPaths=None)
monkeypatch.setattr(psutil, 'cpu_percent', test_cpu_percent)
monkeypatch.setattr(psutil, 'virtual_memory', test_virtual_memory)
monkeypatch.setattr(psutil, 'net_io_counters', test_traffic)
node.monitor.lastKnownTraffic = 0
data1 = node.monitor.captureSystemPerformance()
assert 'cpu' in data1
assert 'ram' in data1
assert 'traffic' in data1
assert data1['cpu']['value'] == cpu
assert data1['ram']['value'] == ram
assert data1['traffic']['value'] == bytes / 1024
cpu = 50
ram = 60
bytes = 2048
assert node.monitor.lastKnownTraffic == data1['traffic']['value']
data2 = node.monitor.captureSystemPerformance()
assert data2['cpu']['value'] == cpu
assert data2['ram']['value'] == ram
assert data2['traffic']['value'] == bytes / 1024 - data1['traffic']['value']
def testNode(pluginManager, tdir, tconf, node_config_helper_class):
name = randomText(20)
nodeReg = genNodeReg(names=[name])
ha, cliname, cliha = nodeReg[name]
config_helper = node_config_helper_class(name, tconf, chroot=tdir)
node = TestNode(name=name, ha=ha, cliname=cliname, cliha=cliha,
nodeRegistry=copy(nodeReg),
config_helper=config_helper,
config=tconf,
primaryDecider=None, pluginPaths=None, seed=randomSeed())
node.start(None)
yield node
node.stop()
def testKeyShareParty(tdir_for_func):
"""
connections to all nodes should be successfully established when key
sharing is enabled.
"""
nodeReg = genNodeReg(5)
logger.debug("-----sharing keys-----")
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
looper.run(checkNodesConnected(nodeSet))
logger.debug("-----key sharing done, connect after key sharing-----")
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as loop:
loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logger.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1, wallet1 = setupClient(looper,
nodeSet,
tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(
eventually(checkRequestReturnedToNode,
node,
wallet1.defaultId,
req.reqId,
instNo,
retryWait=1,
timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testNode(pluginManager, tdir):
name = randomText(20)
nodeReg = genNodeReg(names=[name])
ha, cliname, cliha = nodeReg[name]
node = TestNode(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
nodeRegistry=copy(nodeReg),
basedirpath=tdir,
base_data_dir=tdir,
primaryDecider=None,
pluginPaths=None,
seed=randomSeed())
node.start(None)
yield node
node.stop()
def testKeyShareParty(tdir_for_func):
"""
connections to all nodes should be successfully established when key
sharing is enabled.
"""
nodeReg = genNodeReg(5)
logger.debug("-----sharing keys-----")
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
logger.debug("-----key sharing done, connect after key sharing-----")
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as loop:
loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logger.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
wallet1.defaultId, req.reqId,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def nodeReg(request) -> Dict[str, HA]:
nodeCount = getValueFromModule(request, "nodeCount", 4)
return genNodeReg(count=nodeCount)
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1,
timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
# client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
# request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=timeout))
def assert_msg_count(typ, count):
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
PrePrepare,
1,
retryWait=1,
timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
Prepare,
2,
retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=delayD)) # wait little more than delay
def nodeReg(request) -> Dict[str, HA]:
nodeCount = getValueFromModule(request, "nodeCount", 4)
return genNodeReg(count=nodeCount)
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(
nodeSet, looper, nodeNames[i]) for i in range(
0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
# client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
# request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=timeout))
def assert_msg_count(typ, count):
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, PrePrepare, 1,
retryWait=1, timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(
lambda: assertLength(
getPendingRequestsForReplica(
nodeD.replicas[instNo],
reqType),
0),
retryWait=1,
timeout=delayD)) # wait little more than delay