def __init__(self,
names: Iterable[str] = None,
count: int = None,
nodeReg=None,
tmpdir=None,
keyshare=True,
primaryDecider=None):
self.tmpdir = tmpdir
super().__init__()
self.primaryDecider = primaryDecider
self.nodes = OrderedDict() # type: Dict[str, TestNode]
# Can use just self.nodes rather than maintaining a separate dictionary
# but then have to pluck attributes from the `self.nodes` so keeping
# it simple a the cost of extra memory and its test code so not a big
# deal
if nodeReg:
self.nodeReg = nodeReg
else:
nodeNames = (names if names is not None and count is None else
genNodeNames(count) if count is not None else
error("only one of either names or count is required"))
self.nodeReg = genNodeReg(
names=nodeNames) # type: Dict[str, NodeDetail]
for name in self.nodeReg.keys():
node = self.addNode(name)
# The following lets us access the nodes by name as attributes of the
# NodeSet. It's not a problem unless a node name shadows a member.
self.__dict__.update(self.nodes)
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it will not know
# whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.clientId,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def test_even_compare():
vals = genNodeNames(24)
for v1 in vals:
beats = [v2 for v2 in vals if v1 != v2 and evenCompare(v1, v2)]
print("{} beats {} others: {}".format(v1, len(beats), beats))
evenCompare('Zeta', 'Alpha')
def hashit(s):
b = s.encode('utf-8')
c = crypto_hash_sha256(b)
return c.hex()
for v in vals:
print("{}: {}".format(v, hashit(v)))
def test_distributedConnectionMap():
for nodeCount in range(2, 25):
print("testing for node count: {}".format(nodeCount))
names = genNodeNames(nodeCount)
conmap = distributedConnectionMap(names)
total_combinations = len(list(combinations(names, 2)))
total_combinations_in_map = sum(len(x) for x in conmap.values())
assert total_combinations_in_map == total_combinations
maxPer = math.ceil(total_combinations / nodeCount)
minPer = math.floor(total_combinations / nodeCount)
for x in conmap.values():
assert len(x) <= maxPer
assert len(x) >= minPer
def inner(count=None, names=None) -> Dict[str, NodeDetail]:
"""
:param count: number of nodes, mutually exclusive with names
:param names: iterable with names of nodes, mutually exclusive with count
:return: dictionary of name: (node stack HA, client stack name, client stack HA)
"""
if names is None:
names = genNodeNames(count)
nr = OrderedDict((n, NodeDetail(hagen.prod(), n + CLIENT_STACK_SUFFIX,
hagen.prod())) for n in names)
def extractCliNodeReg(self):
return OrderedDict((n.cliname, n.cliha) for n in self.values())
nr.extractCliNodeReg = types.MethodType(extractCliNodeReg, nr)
return nr
def testConnectWithoutKeySharingFails(tdir_for_func):
"""
attempts at connecting to nodes when key sharing is disabled must fail
"""
nodeNames = genNodeNames(5)
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func,
keyshare=False) as nodes:
with Looper(nodes) as looper:
try:
looper.run(
checkNodesConnected(nodes,
RemoteState(None, None, None)))
except RemoteNotFound:
pass
except KeyError as ex:
assert [n for n in nodeNames
if n == ex.args[0]]
except Exception:
raise
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logging.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary) for a long time.
# TODO: Have a method to ignore a particular message from a node
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
client1.clientId, req.reqId,
req.digest,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers(
tdir_for_func):
"""
A protocol instance must have at least 4 nodes to come up.
The status of the nodes will change from starting to started only after the
addition of the fourth node to the system.
"""
nodeCount = 16
f = 5
minimumNodesToBeUp = 16 - f
nodeNames = genNodeNames(nodeCount)
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
# helpers
def genExpectedStates(connecteds: Iterable[str]):
return {
nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED
for nn in nodeNames}
def checkNodeStatusRemotesAndF(expectedStatus: Status,
nodeIdx: int):
for node in nodeSet.nodes.values():
checkNodeRemotes(node,
genExpectedStates(nodeNames[:nodeIdx + 1]))
assert node.status == expectedStatus
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logging.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(nodeSet, looper, nodeNames[nodeIdx])
looper.run(
eventually(checkNodeStatusRemotesAndF, expectedStatus,
nodeIdx,
retryWait=1, timeout=30))
# tests
logging.debug("Sharing keys")
looper.run(checkNodesConnected(nodeSet))
logging.debug("Remove all the nodes")
for n in nodeNames:
looper.removeProdable(nodeSet.nodes[n])
nodeSet.removeNode(n, shouldClean=False)
logging.debug("Add nodes back one at a time")
for i in range(nodeCount):
nodes = i + 1
if nodes < minimumNodesToBeUp:
expectedStatus = Status.starting
elif nodes < nodeCount:
expectedStatus = Status.started_hungry
else:
expectedStatus = Status.started
addNodeBackAndCheck(i, expectedStatus)