def __can_apply_quorumed(self, batch):
quorum = self._node.quorums.observer_data
batches_for_msg = self._batches[self.seq_no_start(
batch)] # {sender: msg}
num_batches = len(batches_for_msg)
if not quorum.is_reached(len(batches_for_msg)):
logger.debug(
"{} can not apply BATCH with seq_no {} since no quorum yet ({} of {})"
.format(OBSERVER_PREFIX, str(self.seq_no_start(batch)),
num_batches, str(quorum.value)))
return False
msgs = [
json.dumps(msg, sort_keys=True)
for msg in batches_for_msg.values()
]
result, freq = mostCommonElement(msgs)
if not quorum.is_reached(freq):
logger.debug(
"{} can not apply BATCH with seq_no {} since have just {} equal elements ({} needed for quorum)"
.format(OBSERVER_PREFIX, str(self.seq_no_start(batch)), freq,
str(quorum.value)))
return False
return True
def select_primary(self, inst_id: int, prim: Primary):
# If got more than 2f+1 primary declarations then in a position to
# decide whether it is the primary or not `2f + 1` declarations
# are enough because even when all the `f` malicious nodes declare
# a primary, we still have f+1 primary declarations from
# non-malicious nodes. One more assumption is that all the non
# malicious nodes vote for the the same primary
# Find for which node there are maximum primary declarations.
# Cant be a tie among 2 nodes since all the non malicious nodes
# which would be greater than or equal to f+1 would vote for the
# same node
if replica.hasPrimary:
logger.debug("{} Primary already selected; "
"ignoring PRIMARY msg"
.format(replica))
return
if self.hasPrimaryQuorum(inst_id):
if replica.isPrimary is None:
declarations = self.primaryDeclarations[inst_id]
(primary, seqNo), freq = \
mostCommonElement(declarations.values())
logger.display("{}{} selected primary {} for instance {} "
"(view {})"
.format(PRIMARY_SELECTION_PREFIX, replica,
primary, inst_id, self.viewNo),
extra={"cli": "ANNOUNCE",
"tags": ["node-election"]})
logger.debug("{} selected primary on the basis of {}".
format(replica, declarations),
extra={"cli": False})
# If the maximum primary declarations are for this node
# then make it primary
replica.primaryChanged(primary, seqNo)
if inst_id == 0:
self.previous_master_primary = None
# If this replica has nominated itself and since the
# election is over, reset the flag
if self.replicaNominatedForItself == inst_id:
self.replicaNominatedForItself = None
self.node.primary_selected()
self.scheduleElection()
else:
self.discard(prim,
"it already decided primary which is {}".
format(replica.primaryName),
logger.debug)
else:
logger.debug(
"{} received {} but does it not have primary quorum "
"yet".format(self.name, prim))
def select_primary(self, inst_id: int, prim: Primary):
# If got more than 2f+1 primary declarations then in a position to
# decide whether it is the primary or not `2f + 1` declarations
# are enough because even when all the `f` malicious nodes declare
# a primary, we still have f+1 primary declarations from
# non-malicious nodes. One more assumption is that all the non
# malicious nodes vote for the the same primary
# Find for which node there are maximum primary declarations.
# Cant be a tie among 2 nodes since all the non malicious nodes
# which would be greater than or equal to f+1 would vote for the
# same node
if replica.hasPrimary:
logger.debug("{} Primary already selected; "
"ignoring PRIMARY msg"
.format(replica))
return
if self.hasPrimaryQuorum(inst_id):
if replica.isPrimary is None:
declarations = self.primaryDeclarations[inst_id]
(primary, seqNo), freq = \
mostCommonElement(declarations.values())
logger.display("{}{} selected primary {} for instance {} "
"(view {})"
.format(PRIMARY_SELECTION_PREFIX, replica,
primary, inst_id, self.viewNo),
extra={"cli": "ANNOUNCE",
"tags": ["node-election"]})
logger.debug("{} selected primary on the basis of {}".
format(replica, declarations),
extra={"cli": False})
# If the maximum primary declarations are for this node
# then make it primary
replica.primaryChanged(primary, seqNo)
if inst_id == 0:
self.previous_master_primary = None
# If this replica has nominated itself and since the
# election is over, reset the flag
if self.replicaNominatedForItself == inst_id:
self.replicaNominatedForItself = None
self.node.primary_selected()
self.scheduleElection()
else:
self.discard(prim,
"it already decided primary which is {}".
format(replica.primaryName),
logger.debug)
else:
logger.debug(
"{} received {} but does it not have primary quorum "
"yet".format(self.name, prim))
def has_sufficient_same_view_change_done_messages(self) -> Optional[Tuple]:
# Returns whether has a quorum of ViewChangeDone messages that are same
# TODO: Does not look like optimal implementation.
if self._accepted_view_change_done_message is None and \
self._view_change_done:
votes = self._view_change_done.values()
votes = [(nm, tuple(tuple(i) for i in info)) for nm, info in votes]
new_primary, ledger_info = mostCommonElement(votes)
if votes.count((new_primary, ledger_info)) >= self.quorum:
logger.debug('{} found acceptable primary {} and ledger info {}'.
format(self, new_primary, ledger_info))
self._accepted_view_change_done_message = (new_primary,
ledger_info)
else:
logger.debug('{} does not have acceptable primary'.format(self))
return self._accepted_view_change_done_message
def __can_apply_quorumed(self, batch):
quorum = self._node.quorums.observer_data
batches_for_msg = self._batches[self.seq_no_start(batch)] # {sender: msg}
num_batches = len(batches_for_msg)
if not quorum.is_reached(len(batches_for_msg)):
logger.debug("{} can not apply BATCH with seq_no {} since no quorum yet ({} of {})"
.format(OBSERVER_PREFIX, str(self.seq_no_start(batch)), num_batches, str(quorum.value)))
return False
msgs = [json.dumps(msg, sort_keys=True)
for msg in batches_for_msg.values()]
result, freq = mostCommonElement(msgs)
if not quorum.is_reached(freq):
logger.debug(
"{} can not apply BATCH with seq_no {} since have just {} equal elements ({} needed for quorum)"
.format(OBSERVER_PREFIX, str(self.seq_no_start(batch)), freq, str(quorum.value)))
return False
return True
def take_one_quorumed(self, replies, full_req_id):
"""
Checks whether there is sufficint number of equal replies from
different nodes. It uses following logic:
1. Check that there are sufficient replies received at all.
If not - return None.
2. Check that all these replies are equal.
If yes - return one of them.
3. Check that there is a group of equal replies which is large enough.
If yes - return one reply from this group.
4. Return None
"""
if not self.quorums.reply.is_reached(len(replies)):
return None
# excluding state proofs from check since they can be different
def without_state_proof(result):
if STATE_PROOF in result:
result.pop(STATE_PROOF)
return result
results = [without_state_proof(reply["result"])
for reply in replies.values()]
first = results[0]
if all(result == first for result in results):
return first
logger.debug("Received a different result from "
"at least one node for {}"
.format(full_req_id))
result, freq = mostCommonElement(results)
if not self.quorums.reply.is_reached(freq):
return None
return result
def processReelection(self, reelection: Reelection, sender: str):
"""
Process reelection requests sent by other nodes.
If quorum is achieved, proceed with the reelection process.
:param reelection: the reelection request
:param sender: name of the node from which the reelection was sent
"""
logger.debug("{}'s elector started processing reelection msg".format(
self.name))
# Check for election round number to discard any previous
# reelection round message
instId = reelection.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, reelection.instId)
if instId not in self.reElectionProposals:
self.setDefaults(instId)
expectedRoundDiff = 0 if (replica.name
in self.reElectionProposals[instId]) else 1
expectedRound = self.reElectionRounds[instId] + expectedRoundDiff
if not reelection.round == expectedRound:
self.discard(
reelection, "reelection request from {} with round "
"number {} does not match expected {}".format(
sndrRep, reelection.round, expectedRound), logger.debug)
return
if sndrRep not in self.reElectionProposals[instId]:
self.reElectionProposals[instId][sndrRep] = [
tuple(_) for _ in reelection.tieAmong
]
# Check if got reelection messages from at least 2f + 1 nodes (1
# more than max faulty nodes). Necessary because some nodes may
# turn out to be malicious and send re-election frequently
if self.hasReelectionQuorum(instId):
logger.debug("{} achieved reelection quorum".format(replica),
extra={"cli": True})
# Need to find the most frequent tie reported to avoid `tie`s
# from malicious nodes. Since lists are not hashable so
# converting each tie(a list of node names) to a tuple.
ties = [
tuple(t)
for t in self.reElectionProposals[instId].values()
]
tieAmong, freq = mostCommonElement(ties)
self.setElectionDefaults(instId)
if not self.hasPrimaryReplica and not self.was_master_primary_in_prev_view:
# There was a tie among this and some other node(s), so do a
# random wait
if replica.name in [_[0] for _ in tieAmong]:
# Try to nominate self after a random delay but dont block
# until that delay and because a nominate from another
# node might be sent
self._schedule(partial(self.nominateReplica, instId),
random.randint(1, 3))
else:
# Now try to nominate self again as there is a
# reelection
self.nominateReplica(instId)
else:
logger.debug("{} does not have re-election quorum yet. "
"Got only {}".format(
replica,
len(self.reElectionProposals[instId])))
else:
self.discard(
reelection,
"already got re-election proposal from {}".format(sndrRep),
logger.debug)
def processReelection(self, reelection: Reelection, sender: str):
"""
Process reelection requests sent by other nodes.
If quorum is achieved, proceed with the reelection process.
:param reelection: the reelection request
:param sender: name of the node from which the reelection was sent
"""
logger.debug(
"{}'s elector started processing reelection msg".format(self.name))
# Check for election round number to discard any previous
# reelection round message
instId = reelection.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, reelection.instId)
if instId not in self.reElectionProposals:
self.setDefaults(instId)
expectedRoundDiff = 0 if (replica.name in
self.reElectionProposals[instId]) else 1
expectedRound = self.reElectionRounds[instId] + expectedRoundDiff
if not reelection.round == expectedRound:
self.discard(reelection,
"reelection request from {} with round "
"number {} does not match expected {}".
format(sndrRep, reelection.round, expectedRound),
logger.debug)
return
if sndrRep not in self.reElectionProposals[instId]:
self.reElectionProposals[instId][sndrRep] = reelection.tieAmong
# Check if got reelection messages from at least 2f + 1 nodes (1
# more than max faulty nodes). Necessary because some nodes may
# turn out to be malicious and send re-election frequently
if self.hasReelectionQuorum(instId):
logger.debug("{} achieved reelection quorum".format(replica),
extra={"cli": True})
# Need to find the most frequent tie reported to avoid `tie`s
# from malicious nodes. Since lists are not hashable so
# converting each tie(a list of node names) to a tuple.
ties = [tuple(t) for t in
self.reElectionProposals[instId].values()]
tieAmong = mostCommonElement(ties)
self.setElectionDefaults(instId)
# There was a tie among this and some other node(s), so do a
# random wait
if replica.name in tieAmong:
# Try to nominate self after a random delay but dont block
# until that delay and because a nominate from another
# node might be sent
self._schedule(partial(self.nominateReplica, instId),
random.randint(1, 3))
else:
# Now try to nominate self again as there is a reelection
self.nominateReplica(instId)
else:
logger.debug(
"{} does not have re-election quorum yet. Got only {}".format(
replica, len(self.reElectionProposals[instId])))
else:
self.discard(reelection,
"already got re-election proposal from {}".
format(sndrRep),
logger.warning)
def test_mostCommonElement_for_non_hashable_with_hashable_f():
elements = [{1: 2}, {1: 3}, {1: 4}, {1: 3}]
most_common, count = mostCommonElement(elements, lambda el: tuple(el.items()))
assert most_common == {1: 3}
assert count == 2
def test_mostCommonElement_for_non_hashable():
elements = [{1: 2}, {1: 3}, {1: 4}, {1: 3}]
most_common, count = mostCommonElement(elements)
assert most_common == {1: 3}
assert count == 2
def test_mostCommonElement_for_non_hashable_with_hashable_f():
elements = [{1: 2}, {1: 3}, {1: 4}, {1: 3}]
most_common, count = mostCommonElement(elements, lambda el: tuple(el.items()))
assert most_common == {1: 3}
assert count == 2
def test_mostCommonElement_for_mixed_hashable():
elements = [{1: 2}, 4, {1: 3}, 4, {1: 4}, 4, {1: 3}]
most_common, count = mostCommonElement(elements)
assert most_common == 4
assert count == 3
def test_mostCommonElement_for_non_hashable():
elements = [{1: 2}, {1: 3}, {1: 4}, {1: 3}]
most_common, count = mostCommonElement(elements)
assert most_common == {1: 3}
assert count == 2
def test_mostCommonElement_for_hashable():
elements = [1, 2, 3, 3, 2, 5, 2]
most_common, count = mostCommonElement(elements)
assert most_common == 2
assert count == 3
def test_mostCommonElement_for_hashable():
elements = [1, 2, 3, 3, 2, 5, 2]
most_common, count = mostCommonElement(elements)
assert most_common == 2
assert count == 3
def processPrimary(self, prim: Primary, sender: str) -> None:
"""
Process a vote from a replica to select a particular replica as primary.
Once 2f + 1 primary declarations have been received, decide on a primary replica.
:param prim: a vote
:param sender: the name of the node from which this message was sent
"""
logger.debug("{}'s elector started processing primary msg from {} : {}"
.format(self.name, sender, prim))
instId = prim.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, prim.instId)
# Nodes should not be able to declare `Primary` winner more than more
if instId not in self.primaryDeclarations:
self.setDefaults(instId)
if sndrRep not in self.primaryDeclarations[instId]:
self.primaryDeclarations[instId][sndrRep] = prim.name
# If got more than 2f+1 primary declarations then in a position to
# decide whether it is the primary or not `2f + 1` declarations
# are enough because even when all the `f` malicious nodes declare
# a primary, we still have f+1 primary declarations from
# non-malicious nodes. One more assumption is that all the non
# malicious nodes vote for the the same primary
# Find for which node there are maximum primary declarations.
# Cant be a tie among 2 nodes since all the non malicious nodes
# which would be greater than or equal to f+1 would vote for the
# same node
if replica.isPrimary is not None:
logger.debug(
"{} Primary already selected; ignoring PRIMARY msg".format(
replica))
return
if self.hasPrimaryQuorum(instId):
if replica.isPrimary is None:
primary = mostCommonElement(
self.primaryDeclarations[instId].values())
logger.display("{} selected primary {} for instance {} "
"(view {})".
format(replica, primary, instId, self.viewNo),
extra={"cli": "ANNOUNCE"})
logger.debug("{} selected primary on the basis of {}".
format(replica,
self.primaryDeclarations[instId]),
extra={"cli": False})
# If the maximum primary declarations are for this node
# then make it primary
replica.primaryName = primary
# If this replica has nominated itself and since the
# election is over, reset the flag
if self.replicaNominatedForItself == instId:
self.replicaNominatedForItself = None
self.node.primaryFound()
self.scheduleElection()
else:
self.discard(prim,
"it already decided primary which is {}".
format(replica.primaryName),
logger.debug)
else:
logger.debug(
"{} received {} but does it not have primary quorum yet"
.format(self.name, prim))
else:
self.discard(prim,
"already got primary declaration from {}".
format(sndrRep),
logger.warning)
key = (Primary.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
def test_mostCommonElement_for_mixed_hashable():
elements = [{1: 2}, 4, {1: 3}, 4, {1: 4}, 4, {1: 3}]
most_common, count = mostCommonElement(elements)
assert most_common == 4
assert count == 3
