def testDuplicateInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up):
maliciousNode = nodeSet.Alpha
maliciousNode.send(InstanceChange(0))
def chk(instId):
for node in nodeSet:
if node.name != maliciousNode.name:
param = getAllArgs(node, Node.processInstanceChange)
assert param[0]['instChg'].viewNo == instId
assert param[0]['frm'] == maliciousNode.name
looper.run(eventually(chk, 0, retryWait=1, timeout=20))
maliciousNode.send(InstanceChange(0))
def g():
for node in nodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
assert len(
getNodeSuspicions(
node, Suspicions.DUPLICATE_INST_CHNG.code)) == 12
looper.run(eventually(g, retryWait=1, timeout=20))
def testDoNotSendInstChngMsgIfMasterDoesntSeePerformanceProblem(
nodeSet, looper, ensureView):
"""
A node that received an INSTANCE_CHANGE message must not send an
INSTANCE_CHANGE message if it doesn't observe too much difference in
performance between its replicas.
"""
curViewNo = ensureView
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = Node.sendInstanceChange.__name__
for n in nodeSet:
sentInstChanges[n.name] = n.spylog.count(instChngMethodName)
# Send an instance change message to all nodes
icMsg = InstanceChange(curViewNo)
nodeSet.Alpha.send(icMsg)
# Check that that message is discarded.
looper.run(eventually(checkViewNoForNodes, nodeSet, timeout=3))
# No node should have sent a view change and thus must not have called
# `sendInstanceChange`
for n in nodeSet:
assert n.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
def testDiscardInstChngMsgIfMasterDoesntSeePerformanceProblem(
nodeSet, looper, ensureView):
"""
A node that received an INSTANCE_CHANGE message must not send an
INSTANCE_CHANGE message if it doesn't observe too much difference in
performance between its replicas.
"""
curViewNo = ensureView
# Send an instance change message to all nodes
icMsg = InstanceChange(curViewNo)
nodeSet.Alpha.send(icMsg)
# ensure every node but Alpha discards the invalid instance change request
looper.run(
eventually(checkDiscardMsg,
nodeSet,
icMsg,
'did not find the master to be slow',
nodeSet.Alpha,
timeout=5))
# Check that that message is discarded.
looper.run(eventually(checkViewNoForNodes, nodeSet, timeout=3))
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up):
maliciousNode = nodeSet.Alpha
for i in range(0, 5):
maliciousNode.send(InstanceChange(i))
def chk(instId):
for node in nodeSet:
if node.name != maliciousNode.name:
args = getAllArgs(node, Node.processInstanceChange)
assert len(args) == 5
for arg in args:
assert arg['frm'] == maliciousNode.name
for i in range(0, 5):
looper.run(eventually(chk, i, retryWait=1, timeout=20))
def g():
for node in nodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
assert len(
getNodeSuspicions(
node, Suspicions.FREQUENT_INST_CHNG.code)) == 13
looper.run(eventually(g, retryWait=1, timeout=20))
def testInstanceChangeMsgTypeChecking(nodeSet, looper, up):
nodeA = nodeSet.Alpha
nodeB = nodeSet.Beta
ridBeta = nodeA.nodestack.getRemote(nodeB.name).uid
badViewNo = "BAD"
nodeA.send(InstanceChange(badViewNo), ridBeta)
looper.runFor(0.2)
params = nodeB.spylog.getLastParams(TestNode.discard)
def chk():
assert isinstance(params['msg'], InstanceChange)
assert DISCARD_REASON in params['reason']
looper.run(eventually(chk, timeout=5))
def testDiscardInstChngMsgFrmPastView(nodeSet, looper, ensureView):
"""
Once a view change is done, any further INSTANCE_CHANGE messages for that
view must be discarded by the node.
"""
curViewNo = ensureView
# Send an instance change for an old instance message to all nodes
icMsg = InstanceChange(curViewNo - 1)
nodeSet.Alpha.send(icMsg)
# ensure every node but Alpha discards the invalid instance change request
looper.run(
eventually(checkDiscardMsg,
nodeSet,
icMsg,
'less than its view no',
nodeSet.Alpha,
timeout=5))
# Check that that message is discarded.
looper.run(eventually(checkViewNoForNodes, nodeSet, timeout=3))