def checkPrimaryPlacement():
# Node names sorted by rank
sortedNodeNames = sorted(nodeSet.nodes.values(),
key=operator.attrgetter("rank"))
for idx, node in enumerate(sortedNodeNames):
# For instance 0, the primary replica should be on the node with rank 0
if idx == 0:
primaryName = Replica.generateName(sortedNodeNames[idx], 0)
assert node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 1, the primary replica should be on the node with rank 1
if idx == 1:
primaryName = Replica.generateName(sortedNodeNames[idx], 1)
assert not node.replicas[0].isPrimary
assert node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 2, the primary replica should be on the node with rank 2
if idx == 2:
primaryName = Replica.generateName(sortedNodeNames[idx], 2)
assert not node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert node.replicas[2].isPrimary
def getStatus(self):
self.print('Nodes: ', newline=False)
if not self.nodes:
self.print("No nodes are running. Try typing 'new node <name>'.")
else:
self.printNames(self.nodes, newline=True)
if not self.clients:
clients = "No clients are running. Try typing 'new client <name>'."
else:
clients = ",".join(self.clients.keys())
self.print("Clients: " + clients)
f = getMaxFailures(len(self.nodes))
self.print("f-value (number of possible faulty nodes): {}".format(f))
if f != 0 and len(self.nodes) >= 2 * f + 1:
node = list(self.nodes.values())[0]
mPrimary = node.replicas[node.instances.masterId].primaryName
bPrimary = node.replicas[node.instances.backupIds[0]].primaryName
self.print("Instances: {}".format(f + 1))
self.print(" Master (primary is on {})".
format(Replica.getNodeName(mPrimary)))
self.print(" Backup (primary is on {})".
format(Replica.getNodeName(bPrimary)))
else:
self.print("Instances: "
"Not enough nodes to create protocol instances")
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
B.send(Primary(CRep, 0, B.viewNo))
# Node B sends PRIMARY msgs for Node D to all nodes
B.send(Primary(DRep, 0, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug("node {} should have primary declaration for C from node B".format(node))
assert node.elector.primaryDeclarations[0][BRep] == CRep
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def test_ordered_cleaning(tconf):
global_view_no = 2
node = FakeSomething(
name="fake node",
ledger_ids=[0],
viewNo=global_view_no,
)
bls_bft_replica = FakeSomething(
gc=lambda *args: None,
)
replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica)
total = []
num_requests_per_view = 3
for viewNo in range(global_view_no + 1):
for seqNo in range(num_requests_per_view):
reqId = viewNo, seqNo
replica.addToOrdered(*reqId)
total.append(reqId)
# gc is called after stable checkpoint, since no request executed
# in this test starting it manually
replica._gc(100)
# Requests with view lower then previous view
# should not be in ordered
assert len(replica.ordered) == len(total[num_requests_per_view:])
def test_primary_names_cleaning(tconf):
node = FakeSomething(
name="fake node",
ledger_ids=[0],
viewNo=0,
)
bls_bft_replica = FakeSomething(
gc=lambda *args: None,
)
replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica)
replica.primaryName = "Node1:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0")]
node.viewNo += 1
replica.primaryName = "Node2:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0"), (1, "Node2:0")]
node.viewNo += 1
replica.primaryName = "Node3:0"
assert list(replica.primaryNames.items()) == \
[(1, "Node2:0"), (2, "Node3:0")]
node.viewNo += 1
replica.primaryName = "Node4:0"
assert list(replica.primaryNames.items()) == \
[(2, "Node3:0"), (3, "Node4:0")]
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][1]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][1]
def test_msg_len_limit_large_enough_for_preprepare():
config = getConfig()
batch_size = config.Max3PCBatchSize
requests = [Request(signatures={})] * batch_size
req_idr = [req.digest for req in requests]
digest = Replica.batchDigest(requests)
state_root = Base58Serializer().serialize(BLANK_ROOT)
txn_root = Ledger.hashToStr(CompactMerkleTree().root_hash)
pp = PrePrepare(
0,
0,
0,
get_utc_epoch(),
req_idr,
init_discarded(),
digest,
0,
state_root,
txn_root,
0,
True)
assert len(ZStack.serializeMsg(pp)) <= config.MSG_LEN_LIMIT
def checkNomination(node: TestNode, nomineeName: str):
matches = [replica.name for instId, replica in node.elector.replicas.items() if
node.elector.didReplicaNominate(instId) is True and
replica.name in node.elector.nominations[instId] and
node.elector.nominations[instId][replica.name][0] ==
Replica.generateName(nomineeName, instId)]
assert len(matches) > 0
return matches[0]
def getSelfNominationByNode(node: TestNode) -> int:
"""
This function returns the index of the protocol instance for which it nominated itself
@param node: the node
@return: the protocol instance index
"""
for instId, replica in enumerate(node.elector.replicas):
name = Replica.generateName(node.name, instId)
if node.elector.nominations.get(instId, {}).get(name, None) == name:
return instId
def next_primary_replica_name_for_master(self, nodeReg=None):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary. In fact it is not round-robin on
this abstraction layer as currently the primary of master instance is
pointed directly depending on view number, instance id and total
number of nodes.
But since the view number is incremented by 1 before primary selection
then current approach may be treated as round robin.
"""
name = self._next_primary_node_name_for_master(nodeReg)
return name, Replica.generateName(nodeName=name, instId=0)
def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet):
"""
Case 5 - A node making primary declarations for a multiple other nodes.
Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
declares node C as primary to all nodes.
Again node B declares node D as primary to all nodes.
"""
A, B, C, D = txnPoolNodeSet
looper.run(checkNodesConnected(txnPoolNodeSet))
BRep = Replica.generateName(B.name, 0)
CRep = Replica.generateName(C.name, 0)
DRep = Replica.generateName(D.name, 0)
# Node B first sends PRIMARY msgs for Node C to all nodes
# B.send(Primary(CRep, 0, B.viewNo))
B.send(primaryByNode(CRep, B, 0))
# Node B sends PRIMARY msgs for Node D to all nodes
# B.send(Primary(DRep, 0, B.viewNo))
B.send(primaryByNode(DRep, B, 0))
# Ensure elections are done
# also have to take into account the catchup procedure
timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \
delayOfElectionDone
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=timeout)
# All nodes from node A, node C, node D(node B is malicious anyway so not
# considering it) should have primary declarations for node C from node B
# since node B first nominated node C
for node in [A, C, D]:
logger.debug(
"node {} should have primary declaration for C from node B"
.format(node))
assert node.elector.primaryDeclarations[0][BRep][0] == CRep
def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
primaries, nodeReg=None):
"""
Returns name and corresponding instance name of the next node which
is supposed to be a new Primary for backup instance in round-robin
fashion starting from primary of master instance.
"""
if nodeReg is None:
nodeReg = self.node.nodeReg
total_nodes = len(nodeReg)
rank = (master_primary_rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
while name in primaries:
rank = (rank + 1) % total_nodes
name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
return name, Replica.generateName(nodeName=name, instId=instance_id)
def testPrimaryElectionCase1(case1Setup, looper, txnPoolNodeSet):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodeA, nodeB, nodeC, nodeD = txnPoolNodeSet
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(txnPoolNodeSet))
# Node B sends multiple NOMINATE messages for Node D but only after A has
# nominated itself
timeout = waits.expectedPoolNominationTimeout(
nodeCount=len(txnPoolNodeSet))
looper.run(eventually(checkNomination, nodeA, nodeA.name,
retryWait=.25, timeout=timeout))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
# nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
nodeB.nodestack.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert [n[0] for n in node.elector.nominations[instId].values()].count(
Replica.generateName(nodeD.name, instId)) \
<= 1
timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
primaryReplicas = ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet, customTimeout=timeout)
for node in txnPoolNodeSet:
logger.debug(
"{}'s nominations {}".format(node, node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(
instId,
primary.viewNo,
primary.lastPrePrepareSeqNo,
get_utc_epoch(),
[propagated1.digest],
init_discarded(),
Replica.batchDigest([propagated1, ]),
DOMAIN_LEDGER_ID,
primary.stateRootHash(DOMAIN_LEDGER_ID),
primary.txnRootHash(DOMAIN_LEDGER_ID),
0,
True,
primary.stateRootHash(POOL_LEDGER_ID)
)
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(npr, npr.processPrePrepare)
if (param['pre_prepare'][0:3] +
param['pre_prepare'][4:],
param['sender']) == (
expectedPrePrepareRequest[0:3] +
expectedPrePrepareRequest[4:],
primary.name)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes, \
'Non-primary sees correct number pre-prepares - {}'.format(passes)
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodes = keySharedNodes
nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(nodes))
# Node B sends multiple NOMINATE msgs for Node D but only after A has
# nominated itself
looper.run(eventually(checkNomination, nodeA, nodeA.name, retryWait=.25,
timeout=1))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert list(node.elector.nominations[instId].values()).count(
Replica.generateName(nodeD.name, instId)) \
<= 1
primaryReplicas = ensureElectionsDone(looper=looper, nodes=nodes,
retryWait=1, timeout=30)
for node in nodes:
logging.debug(
"{}'s nominations {}".format(node, node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def test_ordered_cleaning(tconf):
global_view_no = 2
node = FakeSomething(
name="fake node",
ledger_ids=[0],
viewNo=global_view_no,
utc_epoch=get_utc_epoch,
get_validators=lambda: [],
db_manager=DatabaseManager(),
requests=[],
mode=Mode.participating,
timer=QueueTimer(),
quorums=Quorums(4),
write_manager=None,
poolManager=FakeSomething(node_names_ordered_by_rank=lambda: [])
)
bls_bft_replica = FakeSomething(
gc=lambda *args: None,
)
replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica)
replica._consensus_data.view_no = global_view_no
total = []
num_requests_per_view = 3
for viewNo in range(global_view_no + 1):
for seqNo in range(num_requests_per_view):
reqId = viewNo, seqNo
replica._ordering_service._add_to_ordered(*reqId)
total.append(reqId)
# gc is called after stable checkpoint, since no request executed
# in this test starting it manually
replica._ordering_service.gc(100)
# Requests with view lower then previous view
# should not be in ordered
assert len(replica._ordering_service.ordered) == len(total[num_requests_per_view:])
def create_pre_prepare_params(state_root,
ledger_id=DOMAIN_LEDGER_ID,
txn_root=None,
timestamp=None,
bls_multi_sig=None,
view_no=0,
pool_state_root=None,
pp_seq_no=0,
inst_id=0,
audit_txn_root=None,
reqs=None):
digest = Replica.batchDigest(reqs) if reqs is not None else "random digest"
req_idrs = [req.key
for req in reqs] if reqs is not None else ["random request"]
params = [
inst_id, view_no, pp_seq_no, timestamp or get_utc_epoch(), req_idrs,
init_discarded(0), digest, ledger_id, state_root, txn_root or '1' * 32,
0, True, pool_state_root or generate_state_root(), audit_txn_root
or generate_state_root()
]
if bls_multi_sig:
params.append(bls_multi_sig.as_list())
return params
def test_msg_len_limit_large_enough_for_preprepare():
config = getConfig()
batch_size = config.Max3PCBatchSize
requests = [Request(signatures={})] * batch_size
req_idr = [req.digest for req in requests]
digest = Replica.batchDigest(requests)
state_root = Base58Serializer().serialize(BLANK_ROOT)
txn_root = Ledger.hashToStr(CompactMerkleTree().root_hash)
pp = PrePrepare(
0,
0,
0,
get_utc_epoch(),
req_idr,
batch_size,
digest,
0,
state_root,
txn_root)
assert len(ZStack.serializeMsg(pp)) <= config.MSG_LEN_LIMIT
def test_primary_names_cleaning(tconf):
node = FakeSomething(
name="fake node",
ledger_ids=[0],
viewNo=0,
utc_epoch=get_utc_epoch,
get_validators=lambda: [],
db_manager=DatabaseManager(),
requests=[],
mode=Mode.participating,
timer=QueueTimer(),
quorums=Quorums(4),
write_manager=None,
poolManager=FakeSomething(node_names_ordered_by_rank=lambda: []),
primaries_selector=RoundRobinConstantNodesPrimariesSelector(
["Alpha", "Beta", "Gamma", "Delta"]))
bls_bft_replica = FakeSomething(gc=lambda *args: None, )
replica = Replica(node,
instId=0,
config=tconf,
bls_bft_replica=bls_bft_replica)
replica.primaryName = "Node1:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node2:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0"), (1, "Node2:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node3:0"
assert list(replica.primaryNames.items()) == \
[(1, "Node2:0"), (2, "Node3:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node4:0"
assert list(replica.primaryNames.items()) == \
[(2, "Node3:0"), (3, "Node4:0")]
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(
instId, primary.viewNo, primary.lastPrePrepareSeqNo,
get_utc_epoch(), [propagated1.digest], init_discarded(),
Replica.batchDigest([
propagated1,
]), DOMAIN_LEDGER_ID, primary.stateRootHash(DOMAIN_LEDGER_ID),
primary.txnRootHash(DOMAIN_LEDGER_ID), 0, True,
primary.stateRootHash(POOL_LEDGER_ID))
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrePrepare)
if (param['pre_prepare'][0:3] + param['pre_prepare'][4:],
param['sender']) == (expectedPrePrepareRequest[0:3] +
expectedPrePrepareRequest[4:],
primary.name)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(
msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes, \
'Non-primary sees correct number pre-prepares - {}'.format(passes)
def test_primary_names_cleaning(tconf):
node = FakeSomething(name="fake node",
ledger_ids=[0],
viewNo=0,
utc_epoch=get_utc_epoch,
get_validators=lambda: [],
internal_bus=InternalBus(),
db_manager=DatabaseManager(),
requests=[],
mode=Mode.participating,
timer=QueueTimer(),
quorums=Quorums(4),
write_manager=None)
bls_bft_replica = FakeSomething(gc=lambda *args: None, )
replica = Replica(node,
instId=0,
config=tconf,
bls_bft_replica=bls_bft_replica)
replica.primaryName = "Node1:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node2:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0"), (1, "Node2:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node3:0"
assert list(replica.primaryNames.items()) == \
[(1, "Node2:0"), (2, "Node3:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node4:0"
assert list(replica.primaryNames.items()) == \
[(2, "Node3:0"), (3, "Node4:0")]
def test_primary_names_cleaning(tconf):
node = FakeSomething(
name="fake node",
ledger_ids=[0],
viewNo=0,
utc_epoch=get_utc_epoch,
get_validators=lambda: [],
)
bls_bft_replica = FakeSomething(gc=lambda *args: None, )
replica = Replica(node,
instId=0,
config=tconf,
bls_bft_replica=bls_bft_replica)
replica.primaryName = "Node1:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node2:0"
assert list(replica.primaryNames.items()) == \
[(0, "Node1:0"), (1, "Node2:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node3:0"
assert list(replica.primaryNames.items()) == \
[(1, "Node2:0"), (2, "Node3:0")]
node.viewNo += 1
replica._consensus_data.view_no = node.viewNo
replica.primaryName = "Node4:0"
assert list(replica.primaryNames.items()) == \
[(2, "Node3:0"), (3, "Node4:0")]
def master_primary_name(self) -> Optional[str]:
nm = self.replicas[0].primaryName
if nm:
return Replica.getNodeName(nm)
def _new_replica(self, instance_id: int, is_master: bool,
bls_bft: BlsBft) -> Replica:
"""
Create a new replica with the specified parameters.
"""
return Replica(self._node, instance_id, is_master, bls_bft)
def master_primary_name(self) -> Optional[str]:
nm = self.replicas[0].primaryName
if nm:
return Replica.getNodeName(nm)
def add_replica(self, inst_id):
self.add_replica_calls.append(inst_id)
self._replicas.update(inst_id=Replica(node=self._node, instId=inst_id))
