def testTestNodeDelay(tdir_for_func):
nodeNames = {"testA", "testB"}
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
nodeA = nodes.getNode("testA")
nodeB = nodes.getNode("testB")
with Looper(nodes) as looper:
for n in nodes:
n.startKeySharing()
logging.debug("connect")
looper.run(checkNodesConnected(nodes))
logging.debug("send one message, without delay")
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
logging.debug(
"set delay, then send another message and find that it doesn't arrive")
msg = randomMsg()
nodeB.nodeIbStasher.delay(delayerMsgTuple(4, type(msg), nodeA.name))
with pytest.raises(AssertionError):
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 3))
logging.debug(
"but then find that it arrives after the delay duration has passed")
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 2))
logging.debug(
"reset the delay, and find another message comes quickly")
nodeB.nodeIbStasher.resetDelays()
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
def testNodesConnectsWhenOneNodeIsLate():
with TemporaryDirectory() as td:
with Looper() as looper:
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = Node(name, nodeReg, basedirpath=td)
looper.add(node)
node.startKeySharing()
nodes.append(node)
for name in names[:3]:
create(name)
looper.run(checkNodesConnected(nodes))
# wait for the election to complete with the first three nodes
looper.runFor(10)
# create the fourth and see that it learns who the primaries are
# from the other nodes
create(names[3])
checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase4(case4Setup, looper):
"""
Case 4 - A node making multiple primary declarations for a particular node.
Consider 4 nodes A, B, C and D. Lets say node B is malicious and is
repeatedly declaring Node D as primary
"""
allNodes = case4Setup
A, B, C, D = allNodes
looper.run(checkNodesConnected(allNodes))
# Node B sends multiple declarations of node D's 0th protocol instance as
# primary to all nodes
for i in range(5):
B.send(Primary(D.name, 0, B.viewNo))
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one primary declaration for node
# D since node D is slow. The one primary declaration for node D,
# that nodes A, C and D might have would be because of node B
def x():
primDecs = list(node.elector.primaryDeclarations[0].values())
assert primDecs.count(D.name) <= 1
for node in (A, C, D):
looper.run(eventually(x, retryWait=0.5, timeout=2))
ensureElectionsDone(looper=looper, nodes=allNodes, retryWait=1, timeout=45)
# Node D should not have any primary replica
assert not D.hasPrimary
def testNodesConnectWhenTheyAllStartAtOnce():
with TemporaryDirectory() as td:
with Looper() as looper:
nodes = []
for name in nodeReg:
node = Node(name, nodeReg, basedirpath=td)
looper.add(node)
node.startKeySharing()
nodes.append(node)
looper.run(checkNodesConnected(nodes))
def testKeyShareParty(tdir_for_func):
"""
connections to all nodes should be successfully established when key
sharing is enabled.
"""
nodeReg = genNodeReg(5)
logging.debug("-----sharing keys-----")
with TestNodeSet(nodeReg=nodeReg,
tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
logging.debug("-----key sharing done, connect after key sharing-----")
with TestNodeSet(nodeReg=nodeReg,
tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as loop:
loop.run(checkNodesConnected(nodeSet),
msgAll(nodeSet))
def testNodesComingUpAtDifferentTimes():
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
with TemporaryDirectory() as td:
print("temporary directory: {}".format(td))
with Looper() as looper:
nodes = []
names = list(nodeReg.keys())
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for i, name in enumerate(names):
node = Node(name, nodeReg, basedirpath=td)
looper.add(node)
node.startKeySharing()
nodes.append(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes,
overrideTimeout=10))
print("connects")
print("node order: {}".format(names))
print("waits: {}".format(waits))
for n in nodes:
n.stop()
for i, n in enumerate(nodes):
n.start()
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes,
overrideTimeout=10))
print("reconnects")
print("node order: {}".format(names))
print("rwaits: {}".format(rwaits))
def testNodeConnection():
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
with TemporaryDirectory() as td:
print("temporary directory: {}".format(td))
with Looper() as looper:
names = ["Alpha", "Beta"]
print(names)
nrg = {n: nodeReg[n] for n in names}
A, B = [Node(name, nrg, basedirpath=td)
for name in names]
looper.add(A)
A.startKeySharing()
looper.runFor(4)
print("wait done")
looper.add(B)
B.startKeySharing()
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start()
looper.runFor(4)
B.start()
looper.run(checkNodesConnected([A, B]))
def testConnectWithoutKeySharingFails(tdir_for_func):
"""
attempts at connecting to nodes when key sharing is disabled must fail
"""
nodeNames = genNodeNames(5)
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func,
keyshare=False) as nodes:
with Looper(nodes) as looper:
try:
looper.run(
checkNodesConnected(nodes,
RemoteState(None, None, None)))
except RemoteNotFound:
pass
except KeyError as ex:
assert [n for n in nodeNames
if n == ex.args[0]]
except Exception:
raise
def pool(looper, nodeSet):
for n in nodeSet: # type: TestNode
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
checkProtocolInstanceSetup(looper, nodeSet, timeout=5)
return adict(looper=looper, nodeset=nodeSet)
def ready(looper, keySharedNodes):
looper.run(checkNodesConnected(keySharedNodes))
return keySharedNodes
def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers(
tdir_for_func):
"""
A protocol instance must have at least 4 nodes to come up.
The status of the nodes will change from starting to started only after the
addition of the fourth node to the system.
"""
nodeCount = 16
f = 5
minimumNodesToBeUp = 16 - f
nodeNames = genNodeNames(nodeCount)
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
# helpers
def genExpectedStates(connecteds: Iterable[str]):
return {
nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED
for nn in nodeNames}
def checkNodeStatusRemotesAndF(expectedStatus: Status,
nodeIdx: int):
for node in nodeSet.nodes.values():
checkNodeRemotes(node,
genExpectedStates(nodeNames[:nodeIdx + 1]))
assert node.status == expectedStatus
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logging.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(nodeSet, looper, nodeNames[nodeIdx])
looper.run(
eventually(checkNodeStatusRemotesAndF, expectedStatus,
nodeIdx,
retryWait=1, timeout=30))
# tests
logging.debug("Sharing keys")
looper.run(checkNodesConnected(nodeSet))
logging.debug("Remove all the nodes")
for n in nodeNames:
looper.removeProdable(nodeSet.nodes[n])
nodeSet.removeNode(n, shouldClean=False)
logging.debug("Add nodes back one at a time")
for i in range(nodeCount):
nodes = i + 1
if nodes < minimumNodesToBeUp:
expectedStatus = Status.starting
elif nodes < nodeCount:
expectedStatus = Status.started_hungry
else:
expectedStatus = Status.started
addNodeBackAndCheck(i, expectedStatus)