def checkProtocolInstanceSetup(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = 1,
customTimeout: float = None,
instances: Sequence[int] = None,
check_primaries=True):
timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes))
checkEveryProtocolInstanceHasOnlyOnePrimary(looper=looper,
nodes=nodes,
retryWait=retryWait,
timeout=timeout,
instances_list=instances)
checkEveryNodeHasAtMostOnePrimary(looper=looper,
nodes=nodes,
retryWait=retryWait,
customTimeout=timeout)
def check_not_in_view_change():
assert all([not n.master_replica._consensus_data.waiting_for_new_view for n in nodes])
looper.run(eventually(check_not_in_view_change, retryWait=retryWait, timeout=customTimeout))
if check_primaries:
for n in nodes[1:]:
assert nodes[0].primaries == n.primaries
primaryReplicas = {replica.instId: replica
for node in nodes
for replica in node.replicas.values() if replica.isPrimary}
return [r[1] for r in
sorted(primaryReplicas.items(), key=operator.itemgetter(0))]
def checkIfSameReplicaIsPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = [r for r in replicas if r.primaryName is None]
assert len(unknowns) == 0, "election should be complete, " \
"but {} out of {} ({}) don't know who the primary " \
"is for protocol instance {}". \
format(len(unknowns), len(replicas), unknowns, replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for " \
"protocol no {}".format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame,
retryWait=retryWait, totalTimeout=timeout))
def setupNodesAndClient(looper: Looper,
nodes: Sequence[TestNode],
nodeReg=None,
tmpdir=None):
looper.run(checkNodesConnected(nodes))
ensureElectionsDone(looper=looper, nodes=nodes)
return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def ensure_node_disconnected(looper: Looper,
disconnected: TestNode,
other_nodes: Iterable[TestNode],
timeout: float = None):
timeout = timeout or (len(other_nodes) - 1)
looper.run(eventually(check_node_disconnected, disconnected,
other_nodes, retryWait=1, timeout=timeout))
def ensure_node_disconnected(looper: Looper,
disconnected: TestNode,
other_nodes: Iterable[TestNode],
timeout: float = None):
timeout = timeout or (len(other_nodes) - 1)
looper.run(eventually(check_node_disconnected, disconnected,
other_nodes, retryWait=1, timeout=timeout))
def checkIfSameReplicaIsPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = [r for r in replicas if r.primaryName is None]
assert len(unknowns) == 0, "election should be complete, " \
"but {} out of {} ({}) don't know who the primary " \
"is for protocol instance {}". \
format(len(unknowns), len(replicas), unknowns, replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for " \
"protocol no {}".format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone,
checkPrisAreOne,
checkPrisAreSame,
retryWait=retryWait,
totalTimeout=timeout))
def testElectionsAfterViewChange(delayedPerf, looper: Looper,
nodeSet: TestNodeSet, up, wallet1, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(
eventually(partial(checkViewChangeInitiatedForNode, node, 1),
retryWait=1,
timeout=20))
# Ensure elections are done again and pool is setup again with appropriate
# protocol instances and each protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, wallet1, client1):
"""
Checking if average latency is being set
"""
for i in range(5):
req = sendRandomRequest(wallet1, client1)
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
for node in nodeSet: # type: Node
mLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId,
node.instances.masterId)
bLat = node.monitor.getAvgLatencyForClient(wallet1.defaultId,
*node.instances.backupIds)
logger.debug(
"Avg. master latency : {}. Avg. backup latency: {}".format(
mLat, bLat))
assert mLat > 0
assert bLat > 0
def checkPoolReady(looper: Looper,
nodes: Sequence[TestNode],
timeout: int = 20):
looper.run(
eventually(checkNodesAreReady,
nodes,
retryWait=.25,
timeout=timeout,
ratchetSteps=10))
def testPostingThroughput(postingStatsEnabled,
decreasedMonitoringTimeouts,
looper: Looper,
nodeSet: TestNodeSet,
wallet1, client1):
"""
The throughput after `DashboardUpdateFreq` seconds and before sending any
requests should be zero.
Send `n` requests in less than `ThroughputWindowSize` seconds and the
throughput till `ThroughputWindowSize` should consider those `n` requests.
After `ThroughputWindowSize` seconds the throughput should be zero
Test `totalRequests` too.
"""
config = decreasedMonitoringTimeouts
# We are sleeping for this window size, because we need to clear previous
# values that were being stored for this much time in tests
looper.runFor(config.ThroughputWindowSize)
reqCount = 10
for node in nodeSet:
assert node.monitor.highResThroughput == 0
assert node.monitor.totalRequests == 0
sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
reqCount,
nodeSet.f)
for node in nodeSet:
assert len(node.monitor.orderedRequestsInLast) == reqCount
assert node.monitor.highResThroughput > 0
assert node.monitor.totalRequests == reqCount
# TODO: Add implementation to actually call firebase plugin
# and test if firebase plugin is sending total request count
# if node is primary
looper.runFor(config.DashboardUpdateFreq)
for node in nodeSet:
node.monitor.spylog.count(Monitor.sendThroughput.__name__) > 0
# Run for latency window duration so that `orderedRequestsInLast`
# becomes empty
looper.runFor(config.ThroughputWindowSize)
def chk():
for node in nodeSet:
assert len(node.monitor.orderedRequestsInLast) == 0
assert node.monitor.highResThroughput == 0
assert node.monitor.totalRequests == reqCount
timeout = config.ThroughputWindowSize
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def prepareNodeSet(looper: Looper, txnPoolNodeSet):
# TODO: Come up with a more specific name for this
# Key sharing party
looper.run(checkNodesConnected(txnPoolNodeSet))
# Remove all the nodes
for n in list(txnPoolNodeSet):
looper.removeProdable(txnPoolNodeSet)
txnPoolNodeSet.remove(n)
def reconnect_node_and_ensure_connected(looper: Looper,
poolNodes: Iterable[TestNode],
connect: Union[str, TestNode],
timeout=None):
if isinstance(connect, TestNode):
connect = connect.name
assert isinstance(connect, str)
reconnectPoolNode(looper, poolNodes, connect)
looper.run(checkNodesConnected(poolNodes, customTimeout=timeout))
def reconnect_node_and_ensure_connected(looper: Looper,
poolNodes: Iterable[TestNode],
connect: Union[str, TestNode],
timeout=None):
if isinstance(connect, TestNode):
connect = connect.name
assert isinstance(connect, str)
reconnectPoolNode(looper, poolNodes, connect)
looper.run(checkNodesConnected(poolNodes, customTimeout=timeout))
def prepareNodeSet(looper: Looper, nodeSet: TestNodeSet):
# TODO: Come up with a more specific name for this
# Key sharing party
looper.run(checkNodesConnected(nodeSet))
# Remove all the nodes
for n in list(nodeSet.nodes.keys()):
looper.removeProdable(nodeSet.nodes[n])
nodeSet.removeNode(n, shouldClean=False)
def prepareNodeSet(looper: Looper, txnPoolNodeSet):
# TODO: Come up with a more specific name for this
# Key sharing party
looper.run(checkNodesConnected(txnPoolNodeSet))
# Remove all the nodes
for n in list(txnPoolNodeSet):
looper.removeProdable(txnPoolNodeSet)
txnPoolNodeSet.remove(n)
def setupNodesAndClient(looper: Looper,
nodes: Sequence[TestNode],
nodeReg=None,
tmpdir=None):
looper.run(checkNodesConnected(nodes))
timeout = 15 + 2 * (len(nodes))
ensureElectionsDone(looper=looper,
nodes=nodes,
retryWait=1,
timeout=timeout)
return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def checkPoolReady(looper: Looper,
nodes: Sequence[TestNode],
customTimeout=None):
"""
Check that pool is in Ready state
"""
timeout = customTimeout or waits.expectedPoolStartUpTimeout(len(nodes))
looper.run(
eventually(checkNodesAreReady, nodes,
retryWait=.25,
timeout=timeout,
ratchetSteps=10))
def checkPoolReady(looper: Looper,
nodes: Sequence[TestNode],
customTimeout=None):
"""
Check that pool is in Ready state
"""
timeout = customTimeout or waits.expectedPoolStartUpTimeout(len(nodes))
looper.run(
eventually(checkNodesAreReady, nodes,
retryWait=.25,
timeout=timeout,
ratchetSteps=10))
def checkEveryNodeHasAtMostOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
customTimeout: float = None):
def checkAtMostOnePrim(node):
prims = [r for r in node.replicas.values() if r.isPrimary]
assert len(prims) <= 1
timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes))
for node in nodes:
looper.run(eventually(checkAtMostOnePrim,
node,
retryWait=retryWait,
timeout=timeout))
def checkEveryNodeHasAtMostOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
customTimeout: float = None):
def checkAtMostOnePrim(node):
prims = [r for r in node.replicas if r.isPrimary]
assert len(prims) <= 1
timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes))
for node in nodes:
looper.run(eventually(checkAtMostOnePrim,
node,
retryWait=retryWait,
timeout=timeout))
def setupClient(looper: Looper,
nodes: Sequence[TestNode] = None,
nodeReg=None,
tmpdir=None,
identifier=None,
verkey=None):
client1, wallet = genTestClient(nodes=nodes,
nodeReg=nodeReg,
tmpdir=tmpdir,
identifier=identifier,
verkey=verkey)
looper.add(client1)
looper.run(client1.ensureConnectedToNodes())
return client1, wallet
def checkEveryNodeHasAtMostOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
timeout: float = None):
def checkAtMostOnePrim(node):
prims = [r for r in node.replicas if r.isPrimary]
assert len(prims) <= 1
for node in nodes:
looper.run(
eventually(checkAtMostOnePrim,
node,
retryWait=retryWait,
timeout=timeout))
def testPostingLatency(postingStatsEnabled,
decreasedMonitoringTimeouts,
looper: Looper,
nodeSet: TestNodeSet,
wallet1, client1):
"""
The latencies (master as well as average of backups) after
`DashboardUpdateFreq` seconds and before sending any requests should be zero.
Send `n` requests in less than `LatencyWindowSize` seconds and the
latency till `LatencyWindowSize` should consider those `n` requests.
After `LatencyWindowSize` seconds the latencies should be zero
"""
config = decreasedMonitoringTimeouts
# Run for latency window duration so that `latenciesByMasterInLast` and
# `latenciesByBackupsInLast` become empty
looper.runFor(config.LatencyWindowSize)
reqCount = 10
for node in nodeSet:
assert node.monitor.masterLatency == 0
assert node.monitor.avgBackupLatency == 0
sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
reqCount,
nodeSet.f)
for node in nodeSet:
assert node.monitor.masterLatency > 0
assert node.monitor.avgBackupLatency > 0
looper.runFor(config.DashboardUpdateFreq)
for node in nodeSet:
node.monitor.spylog.count(Monitor.sendLatencies.__name__) > 0
# Run for latency window duration so that `latenciesByMasterInLast` and
# `latenciesByBackupsInLast` become empty
looper.runFor(config.LatencyWindowSize)
def chk():
for node in nodeSet:
assert node.monitor.masterLatency == 0
assert node.monitor.avgBackupLatency == 0
timeout = config.LatencyWindowSize
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def wait_for_elections_done_on_given_nodes(looper: Looper,
nodes: Iterable[Node],
num_of_instances: int,
timeout: float,
retry_wait: float=1.0):
"""
Wait for primary elections to be completed on all the replicas
of the given nodes.
"""
def check_num_of_replicas():
for node in nodes:
assert len(node.replicas) == num_of_instances
def verify_each_replica_knows_its_primary():
for node in nodes:
for inst_id, replica in node.replicas.items():
assert replica.hasPrimary
looper.run(eventuallyAll(check_num_of_replicas,
verify_each_replica_knows_its_primary,
totalTimeout=timeout,
retryWait=retry_wait))
def wait_for_elections_done_on_given_nodes(looper: Looper,
nodes: Iterable[Node],
num_of_instances: int,
timeout: float,
retry_wait: float=1.0):
"""
Wait for primary elections to be completed on all the replicas
of the given nodes.
"""
def check_num_of_replicas():
for node in nodes:
assert len(node.replicas) == num_of_instances
def verify_each_replica_knows_its_primary():
for node in nodes:
for inst_id, replica in node.replicas.items():
assert replica.hasPrimary
looper.run(eventuallyAll(check_num_of_replicas,
verify_each_replica_knows_its_primary,
totalTimeout=timeout,
retryWait=retry_wait))
class UserScenario(metaclass=ABCMeta):
def __init__(self, seed, logFileName=None):
if logFileName:
Logger().enableFileLogging(logFileName)
self._seed = seed
self._client = None
self._wallet = None
self._looper = None
@property
def identifier(self):
if self._wallet:
return self._wallet.defaultId
else:
return None
@property
def verkey(self):
if self._wallet:
return self._wallet.getVerkey()
else:
return None
@classmethod
def runInstance(cls, *args, **kwargs):
cls(*args, **kwargs).run()
def run(self):
try:
self._createClientAndWallet()
self._looper = Looper(debug=getConfig().LOOPER_DEBUG)
try:
self._startClient()
self.do()
finally:
self._looper.shutdownSync()
self._looper = None
except BaseException as ex:
logger.exception(
"User scenario throws out exception: {}".format(ex),
exc_info=ex)
raise ex
@abstractmethod
def do(self):
pass
def performOperation(self, op):
req = self._wallet.signOp(op)
self._client.submitReqs(req)
def getRequestResult(reqKey):
reply, error = self._client.replyIfConsensus(*reqKey)
if reply is None and error is None:
raise Exception("Request has not been completed yet")
else:
return reply, error
reply, error = self._looper.run(
eventually(partial(getRequestResult, req.key),
retryWait=.5,
timeout=5))
assert not error, error
if reply[DATA]:
result = json.loads(reply[DATA])
else:
result = None
return result
def generateNewSigner(self):
assert self.identifier
return SimpleSigner(identifier=self.identifier)
def changeSigner(self, newSigner):
assert newSigner.identifier == self.identifier
self._wallet.updateSigner(self.identifier, newSigner)
logger.info("Changed signer. New verkey: {}".format(self.verkey))
def _createClientAndWallet(self):
signer = SimpleSigner(seed=self._seed)
port = genHa()[1]
ha = HA('0.0.0.0', port)
self._client = Client(name=signer.identifier, ha=ha)
self._wallet = Wallet(name=signer.identifier)
self._wallet.addIdentifier(signer=signer)
logger.info("Identifier: {}".format(self.identifier))
logger.info("Signer's verkey: {}".format(self.verkey))
def _startClient(self):
self._looper.add(self._client)
def ensureConnectedToAll():
connectedNodes = self._client.nodestack.connecteds
connectedNodesNum = len(connectedNodes)
totalNodes = len(self._client.nodeReg)
logger.info("Connected {} / {} nodes".format(
connectedNodesNum, totalNodes))
for node in connectedNodes:
logger.info(" {}".format(node))
if connectedNodesNum == 0:
raise Exception("Not connected to any")
elif connectedNodesNum < totalNodes * 0.8:
raise Exception("Not connected fully")
else:
return True
self._looper.run(
eventually(ensureConnectedToAll, retryWait=.5, timeout=5))
