def checkPrimaryPlacement():
# Node names sorted by rank
sortedNodeNames = sorted(nodeSet.nodes.values(),
key=operator.attrgetter("rank"))
for idx, node in enumerate(sortedNodeNames):
# For instance 0, the primary replica should be on the node with rank 0
if idx == 0:
primaryName = Replica.generateName(sortedNodeNames[idx], 0)
assert node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 1, the primary replica should be on the node with rank 1
if idx == 1:
primaryName = Replica.generateName(sortedNodeNames[idx], 1)
assert not node.replicas[0].isPrimary
assert node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 2, the primary replica should be on the node with rank 2
if idx == 2:
primaryName = Replica.generateName(sortedNodeNames[idx], 2)
assert not node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert node.replicas[2].isPrimary
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25,
timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
B.send(Primary(CRep, 0, B.viewNo))
# Node B sends PRIMARY msgs for Node D to all nodes
B.send(Primary(DRep, 0, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug("node {} should have primary declaration for C from node B".format(node))
assert node.elector.primaryDeclarations[0][BRep] == CRep
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
B.send(Primary(CRep, 0, B.viewNo))
# Node B sends PRIMARY msgs for Node D to all nodes
B.send(Primary(DRep, 0, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug(
"node {} should have primary declaration for C from node B"
.format(node))
assert node.elector.primaryDeclarations[0][BRep] == CRep
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][1]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][1]
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][0]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][0]
def checkNomination(node: TestNode, nomineeName: str):
matches = [replica.name for instId, replica in node.elector.replicas.items() if
node.elector.didReplicaNominate(instId) is True and
replica.name in node.elector.nominations[instId] and
node.elector.nominations[instId][replica.name][0] ==
Replica.generateName(nomineeName, instId)]
assert len(matches) > 0
return matches[0]
def next_primary_replica_name(self, instance_id):
"""
Returns name of the next node which is supposed to be a new Primary
in round-robin fashion
"""
return Replica.generateName(
nodeName=self.next_primary_node_name(instance_id),
instId=instance_id)
def checkNomination(node: TestNode, nomineeName: str):
matches = [replica.name for instId, replica in enumerate(node.elector.replicas) if
node.elector.didReplicaNominate(instId) is True and
replica.name in node.elector.nominations[instId] and
node.elector.nominations[instId][replica.name][0] ==
Replica.generateName(nomineeName, instId)]
assert len(matches) > 0
return matches[0]
def getSelfNominationByNode(node: TestNode) -> int:
"""
This function returns the index of the protocol instance for which it nominated itself
@param node: the node
@return: the protocol instance index
"""
for instId, replica in enumerate(node.elector.replicas):
name = Replica.generateName(node.name, instId)
if node.elector.nominations.get(instId, {}).get(name, None) == name:
return instId
def getSelfNominationByNode(node: TestNode) -> int:
"""
This function returns the index of the protocol instance for which it nominated itself
@param node: the node
@return: the protocol instance index
"""
for instId, replica in enumerate(node.elector.replicas):
name = Replica.generateName(node.name, instId)
if node.elector.nominations.get(instId, {}).get(name, None) == name:
return instId
def next_primary_replica_name_for_master(self, nodeReg=None):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary. In fact it is not round-robin on
this abstraction layer as currently the primary of master instance is
pointed directly depending on view number, instance id and total
number of nodes.
But since the view number is incremented by 1 before primary selection
then current approach may be treated as round robin.
"""
name = self._next_primary_node_name_for_master(nodeReg)
return name, Replica.generateName(nodeName=name, instId=0)
def next_primary_replica_name_for_master(self, nodeReg=None):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary. In fact it is not round-robin on
this abstraction layer as currently the primary of master instance is
pointed directly depending on view number, instance id and total
number of nodes.
But since the view number is incremented by 1 before primary selection
then current approach may be treated as round robin.
"""
name = self._next_primary_node_name_for_master(nodeReg)
return name, Replica.generateName(nodeName=name, instId=0)
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodes = keySharedNodes
nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(nodes))
# Node B sends multiple NOMINATE messages for Node D but only after A has
# nominated itself
timeout = waits.expectedPoolNominationTimeout(
nodeCount=len(keySharedNodes))
looper.run(
eventually(checkNomination,
nodeA,
nodeA.name,
retryWait=.25,
timeout=timeout))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
# nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
nodeB.nodestack.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert [n[0] for n in node.elector.nominations[instId].values()].count(
Replica.generateName(nodeD.name, instId)) \
<= 1
timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
primaryReplicas = ensureElectionsDone(looper=looper,
nodes=nodes,
customTimeout=timeout)
for node in nodes:
logger.debug("{}'s nominations {}".format(node,
node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
A, B, C, D = txnPoolNodeSet
looper.run(checkNodesConnected(txnPoolNodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
# B.send(Primary(CRep, 0, B.viewNo))
B.send(primaryByNode(CRep, B, 0))
# Node B sends PRIMARY msgs for Node D to all nodes
# B.send(Primary(DRep, 0, B.viewNo))
B.send(primaryByNode(DRep, B, 0))
# Ensure elections are done
# also have to take into account the catchup procedure
timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \
delayOfElectionDone
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=timeout)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug(
"node {} should have primary declaration for C from node B".format(
node))
assert node.elector.primaryDeclarations[0][BRep][0] == CRep
def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
A, B, C, D = txnPoolNodeSet
looper.run(checkNodesConnected(txnPoolNodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
# B.send(Primary(CRep, 0, B.viewNo))
B.send(primaryByNode(CRep, B, 0))
# Node B sends PRIMARY msgs for Node D to all nodes
# B.send(Primary(DRep, 0, B.viewNo))
B.send(primaryByNode(DRep, B, 0))
# Ensure elections are done
# also have to take into account the catchup procedure
timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \
delayOfElectionDone
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=timeout)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug(
"node {} should have primary declaration for C from node B"
.format(node))
assert node.elector.primaryDeclarations[0][BRep][0] == CRep
def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
primaries, node_reg, node_ids):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary for backup instance in round-robin
fashion starting from primary of master instance.
"""
if node_reg is None:
node_reg = self.node.nodeReg
total_nodes = len(node_reg)
rank = (master_primary_rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, node_reg, node_ids)
while name in primaries:
rank = (rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, node_reg, node_ids)
return name, Replica.generateName(nodeName=name, instId=instance_id)
def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
primaries, nodeReg=None):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary for backup instance in round-robin
fashion starting from primary of master instance.
"""
if nodeReg is None:
nodeReg = self.node.nodeReg
total_nodes = len(nodeReg)
rank = (master_primary_rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
while name in primaries:
rank = (rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
return name, Replica.generateName(nodeName=name, instId=instance_id)
def testPrimaryElectionCase1(case1Setup, looper, txnPoolNodeSet):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodeA, nodeB, nodeC, nodeD = txnPoolNodeSet
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(txnPoolNodeSet))
# Node B sends multiple NOMINATE messages for Node D but only after A has
# nominated itself
timeout = waits.expectedPoolNominationTimeout(
nodeCount=len(txnPoolNodeSet))
looper.run(eventually(checkNomination, nodeA, nodeA.name,
retryWait=.25, timeout=timeout))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
# nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
nodeB.nodestack.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert [n[0] for n in node.elector.nominations[instId].values()].count(
Replica.generateName(nodeD.name, instId)) \
<= 1
timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
primaryReplicas = ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet, customTimeout=timeout)
for node in txnPoolNodeSet:
logger.debug(
"{}'s nominations {}".format(node, node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodes = keySharedNodes
nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(nodes))
# Node B sends multiple NOMINATE msgs for Node D but only after A has
# nominated itself
looper.run(eventually(checkNomination, nodeA, nodeA.name, retryWait=.25,
timeout=1))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert list(node.elector.nominations[instId].values()).count(
Replica.generateName(nodeD.name, instId)) \
<= 1
primaryReplicas = ensureElectionsDone(looper=looper, nodes=nodes,
retryWait=1, timeout=30)
for node in nodes:
logging.debug(
"{}'s nominations {}".format(node, node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def on_catchup_complete(self):
# Select primaries after usual catchup (not view change)
ledger = self.node.getLedger(AUDIT_LEDGER_ID)
self.node.backup_instance_faulty_processor.restore_replicas()
self.node.drop_primaries()
if len(ledger) == 0:
self.node.select_primaries()
else:
# Emulate view change start
self.node.view_changer.previous_view_no = self.node.viewNo
self.node.viewNo = get_payload_data(ledger.get_last_committed_txn())[AUDIT_TXN_VIEW_NO]
self.node.view_changer.previous_master_primary = self.node.master_primary_name
self.node.view_changer.set_defaults()
self.node.primaries = self._get_last_audited_primaries()
if len(self.replicas) != len(self.node.primaries):
logger.error('Audit ledger has inconsistent number of nodes. '
'Node primaries = {}'.format(self.node.primaries))
if any(p not in self.node.nodeReg for p in self.node.primaries):
logger.error('Audit ledger has inconsistent names of primaries. '
'Node primaries = {}'.format(self.node.primaries))
# Similar functionality to select_primaries
for instance_id, replica in self.replicas.items():
if instance_id == 0:
self.node.start_participating()
replica.primaryChanged(
Replica.generateName(self.node.primaries[instance_id], instance_id))
self.node.primary_selected(instance_id)
# Primary propagation
last_sent_pp_seq_no_restored = False
for replica in self.replicas.values():
replica.on_propagate_primary_done()
if self.node.view_changer.previous_view_no == 0:
last_sent_pp_seq_no_restored = \
self.node.last_sent_pp_store_helper.try_restore_last_sent_pp_seq_no()
if not last_sent_pp_seq_no_restored:
self.node.last_sent_pp_store_helper.erase_last_sent_pp_seq_no()
# Emulate view_change ending
self.node.on_view_propagated()
