def checkIfSameReplicaIPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = sum(1 for r in replicas if r.isPrimary is None)
assert unknowns == 0, "election should be complete, but {} out of {} " \
"don't know who the primary is for protocol no {}" \
.format(unknowns, len(replicas), replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for protocol no {}" \
.format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame,
retryWait=retryWait, totalTimeout=timeout))
def setupNodesAndClient(looper: Looper, nodes: Sequence[TestNode], nodeReg=None,
tmpdir=None):
looper.run(checkNodesConnected(nodes))
timeout = 15 + 2 * (len(nodes))
ensureElectionsDone(looper=looper, nodes=nodes, retryWait=1,
timeout=timeout)
return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def setupNodesAndClientAndSendRandomReq(looper: Looper,
nodes: Sequence[TestNode], nodeReg=None,
tmpdir=None):
_client = setupNodesAndClient(looper, nodes, nodeReg, tmpdir)
request = sendRandomRequest(_client)
timeout = 3 * len(nodes)
looper.run(eventually(checkSufficientRepliesRecvd, _client.inBox,
request.reqId, 1,
retryWait=1, timeout=timeout))
return _client, request
def setupClient(looper: Looper,
nodes: Sequence[TestNode] = None,
nodeReg=None,
tmpdir=None):
client1 = genTestClient(nodes=nodes,
nodeReg=nodeReg,
tmpdir=tmpdir)
looper.add(client1)
looper.run(client1.ensureConnectedToNodes())
return client1
def sendReqsToNodesAndVerifySuffReplies(looper: Looper, client: TestClient,
numReqs: int, timeout: float = None):
nodeCount = len(client.nodeReg)
f = getMaxFailures(nodeCount)
requests = [sendRandomRequest(client) for i in range(numReqs)]
for request in requests:
looper.run(eventually(checkSufficientRepliesRecvd, client.inBox,
request.reqId, f,
retryWait=1, timeout=3 * nodeCount))
return requests
def prepareNodeSet(looper: Looper, nodeSet: TestNodeSet):
# TODO: Come up with a more specific name for this
for n in nodeSet:
n.startKeySharing()
# Key sharing party
looper.run(checkNodesConnected(nodeSet))
# Remove all the nodes
for n in list(nodeSet.nodes.keys()):
looper.removeProdable(nodeSet.nodes[n])
nodeSet.removeNode(n, shouldClean=False)
async def aSetupClient(looper: Looper,
nodes: Sequence[TestNode] = None,
nodeReg=None,
tmpdir=None):
"""
async version of above
"""
client1 = genTestClient(nodes=nodes,
nodeReg=nodeReg,
tmpdir=tmpdir)
looper.add(client1)
await client1.ensureConnectedToNodes()
return client1
def checkEveryNodeHasAtMostOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
timeout: float = None):
def checkAtMostOnePrim(node):
prims = [r for r in node.replicas if r.isPrimary]
assert len(prims) <= 1
for node in nodes:
looper.run(eventually(checkAtMostOnePrim,
node,
retryWait=retryWait,
timeout=timeout))
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20))
# Ensure elections are done again and pool is setup again with appropriate protocol instances and each
# protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, client1):
"""
Checking if average latency is being set
"""
for i in range(5):
req = sendRandomRequest(client1)
looper.run(eventually(checkSufficientRepliesRecvd,
client1.inBox, req.reqId, 1,
retryWait=1, timeout=5))
for node in nodeSet:
mLat = node.monitor.getAvgLatencyForClient(client1.clientId,
node.masterInst)
bLat = node.monitor.getAvgLatencyForClient(client1.clientId,
*node.nonMasterInsts)
logging.debug("Avg. master latency : {}. Avg. backup latency: {}".
format(mLat, bLat))
assert mLat > 0
assert bLat > 0
def checkPoolReady(looper: Looper, nodes: Sequence[TestNode],
timeout: int = 20):
looper.run(
eventually(checkNodesAreReady, nodes, retryWait=.25,
timeout=timeout,
ratchetSteps=10))
def addNodeBack(nodeSet: TestNodeSet,
looper: Looper,
nodeName: str) -> TestNode:
node = nodeSet.addNode(nodeName)
looper.add(node)
return node