def processNominate(self, nom: Nomination, sender: str):
"""
Process a Nomination message.
:param nom: the nomination message
:param sender: sender address of the nomination
"""
logger.debug("{} elector started processing nominate msg: {}".format(
self.name, nom))
instId = nom.instId
replica = self.replicas[instId]
if instId == 0 and replica.getNodeName(
nom.name) == self.previous_master_primary:
self.discard(nom,
'{} got Nomination from {} for {} who was primary'
' of master in previous view too'.format(
self, sender, nom.name),
logMethod=logger.debug)
return False
sndrRep = replica.generateName(sender, nom.instId)
if not self.didReplicaNominate(instId):
if instId not in self.nominations:
self.setDefaults(instId)
self.nominations[instId][replica.name] = (nom.name, nom.ordSeqNo)
self.sendNomination(nom.name, nom.instId, nom.viewNo, nom.ordSeqNo)
logger.debug("{} nominating {} for instance {}".format(
replica, nom.name, nom.instId),
extra={
"cli": "PLAIN",
"tags": ["node-nomination"]
})
else:
logger.debug("{} already nominated".format(replica.name))
# Nodes should not be able to vote more than once
if sndrRep not in self.nominations[instId]:
self.nominations[instId][sndrRep] = (nom.name, nom.ordSeqNo)
logger.debug("{} attempting to decide primary based on nomination "
"request: {} from {}".format(replica, nom, sndrRep))
self._schedule(partial(self.decidePrimary, instId))
else:
self.discard(nom, "already got nomination from {}".format(sndrRep),
logger.debug)
key = (Nomination.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
def processNominate(self, nom: Nomination, sender: str):
"""
Process a Nomination message.
:param nom: the nomination message
:param sender: sender address of the nomination
"""
logger.debug("{} elector started processing nominate msg: {}".
format(self.name, nom))
instId = nom.instId
replica = self.replicas[instId]
if instId == 0 and replica.getNodeName(
nom.name) == self.previous_master_primary:
self.discard(
nom, '{} got Nomination from {} for {} who was primary'
' of master in previous view too'. format(
self, sender, nom.name), logMethod=logger.debug)
return False
sndrRep = replica.generateName(sender, nom.instId)
if not self.didReplicaNominate(instId):
if instId not in self.nominations:
self.setDefaults(instId)
self.nominations[instId][replica.name] = (nom.name, nom.ordSeqNo)
self.sendNomination(nom.name, nom.instId, nom.viewNo,
nom.ordSeqNo)
logger.debug("{} nominating {} for instance {}".
format(replica, nom.name, nom.instId),
extra={"cli": "PLAIN", "tags": ["node-nomination"]})
else:
logger.debug("{} already nominated".format(replica.name))
# Nodes should not be able to vote more than once
if sndrRep not in self.nominations[instId]:
self.nominations[instId][sndrRep] = (nom.name, nom.ordSeqNo)
logger.debug("{} attempting to decide primary based on nomination "
"request: {} from {}".format(replica, nom, sndrRep))
self._schedule(partial(self.decidePrimary, instId))
else:
self.discard(nom,
"already got nomination from {}".
format(sndrRep),
logger.debug)
key = (Nomination.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
def processPrimary(self, prim: Primary, sender: str) -> None:
"""
Process a vote from a replica to select a particular replica as primary.
Once 2f + 1 primary declarations have been received, decide on a
primary replica.
:param prim: a vote
:param sender: the name of the node from which this message was sent
"""
logger.debug(
"{}'s elector started processing primary msg from {} : {}".format(
self.name, sender, prim))
instId = prim.instId
replica = self.replicas[instId]
if instId == 0 and replica.getNodeName(
prim.name) == self.previous_master_primary:
self.discard(prim,
'{} got Primary from {} for {} who was primary'
' of master in previous view too'.format(
self, sender, prim.name),
logMethod=logger.info)
return
sndrRep = replica.generateName(sender, prim.instId)
# Nodes should not be able to declare `Primary` winner more than more
if instId not in self.primaryDeclarations:
self.setDefaults(instId)
if sndrRep not in self.primaryDeclarations[instId]:
self.primaryDeclarations[instId][sndrRep] = (prim.name,
prim.ordSeqNo)
self.select_primary(instId, prim)
else:
self.discard(
prim,
"already got primary declaration from {}".format(sndrRep),
logger.debug)
key = (Primary.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
def processNominate(self, nom: Nomination, sender: str):
"""
Process a Nomination message.
:param nom: the nomination message
:param sender: sender address of the nomination
"""
logger.debug("{}'s elector started processing nominate msg: {}".format(
self.name, nom))
instId = nom.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, nom.instId)
if not self.didReplicaNominate(instId):
if instId not in self.nominations:
self.setDefaults(instId)
self.nominations[instId][replica.name] = nom.name
self.sendNomination(nom.name, nom.instId, nom.viewNo)
logger.debug("{} nominating {} for instance {}".format(
replica, nom.name, nom.instId),
extra={"cli": "PLAIN"})
else:
logger.debug("{} already nominated".format(replica.name))
# Nodes should not be able to vote more than once
if sndrRep not in self.nominations[instId]:
self.nominations[instId][sndrRep] = nom.name
logger.debug("{} attempting to decide primary based on nomination "
"request: {} from {}".format(replica, nom, sndrRep))
self._schedule(partial(self.decidePrimary, instId))
else:
self.discard(
nom, "already got nomination from {}".format(replica, sndrRep),
logger.warning)
key = (Nomination.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
# If got more than one duplicate message then blacklist
if self.duplicateMsgs[key] > 1:
self.send(
BlacklistMsg(Suspicions.DUPLICATE_NOM_SENT.code, sender))
def processNominate(self, nom: Nomination, sender: str):
"""
Process a Nomination message.
:param nom: the nomination message
:param sender: sender address of the nomination
"""
logger.debug("{}'s elector started processing nominate msg: {}".
format(self.name, nom))
instId = nom.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, nom.instId)
if not self.didReplicaNominate(instId):
if instId not in self.nominations:
self.setDefaults(instId)
self.nominations[instId][replica.name] = nom.name
self.sendNomination(nom.name, nom.instId, nom.viewNo)
logger.debug("{} nominating {} for instance {}".
format(replica, nom.name, nom.instId),
extra={"cli": "PLAIN"})
else:
logger.debug("{} already nominated".format(replica.name))
# Nodes should not be able to vote more than once
if sndrRep not in self.nominations[instId]:
self.nominations[instId][sndrRep] = nom.name
logger.debug("{} attempting to decide primary based on nomination "
"request: {} from {}".format(replica, nom, sndrRep))
self._schedule(partial(self.decidePrimary, instId))
else:
self.discard(nom,
"already got nomination from {}".
format(replica, sndrRep),
logger.warning)
key = (Nomination.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
# If got more than one duplicate message then blacklist
if self.duplicateMsgs[key] > 1:
self.send(BlacklistMsg(Suspicions.DUPLICATE_NOM_SENT.code, sender))
def processPrimary(self, prim: Primary, sender: str) -> None:
"""
Process a vote from a replica to select a particular replica as primary.
Once 2f + 1 primary declarations have been received, decide on a
primary replica.
:param prim: a vote
:param sender: the name of the node from which this message was sent
"""
logger.debug("{}'s elector started processing primary msg from {} : {}"
.format(self.name, sender, prim))
instId = prim.instId
replica = self.replicas[instId]
if instId == 0 and replica.getNodeName(
prim.name) == self.previous_master_primary:
self.discard(prim, '{} got Primary from {} for {} who was primary'
' of master in previous view too'.
format(self, sender, prim.name),
logMethod=logger.info)
return
sndrRep = replica.generateName(sender, prim.instId)
# Nodes should not be able to declare `Primary` winner more than more
if instId not in self.primaryDeclarations:
self.setDefaults(instId)
if sndrRep not in self.primaryDeclarations[instId]:
self.primaryDeclarations[instId][sndrRep] = (prim.name,
prim.ordSeqNo)
self.select_primary(instId, prim)
else:
self.discard(prim,
"already got primary declaration from {}".
format(sndrRep),
logger.debug)
key = (Primary.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1
def processReelection(self, reelection: Reelection, sender: str):
"""
Process reelection requests sent by other nodes.
If quorum is achieved, proceed with the reelection process.
:param reelection: the reelection request
:param sender: name of the node from which the reelection was sent
"""
logger.debug("{}'s elector started processing reelection msg".format(
self.name))
# Check for election round number to discard any previous
# reelection round message
instId = reelection.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, reelection.instId)
if instId not in self.reElectionProposals:
self.setDefaults(instId)
expectedRoundDiff = 0 if (replica.name
in self.reElectionProposals[instId]) else 1
expectedRound = self.reElectionRounds[instId] + expectedRoundDiff
if not reelection.round == expectedRound:
self.discard(
reelection, "reelection request from {} with round "
"number {} does not match expected {}".format(
sndrRep, reelection.round, expectedRound), logger.debug)
return
if sndrRep not in self.reElectionProposals[instId]:
self.reElectionProposals[instId][sndrRep] = [
tuple(_) for _ in reelection.tieAmong
]
# Check if got reelection messages from at least 2f + 1 nodes (1
# more than max faulty nodes). Necessary because some nodes may
# turn out to be malicious and send re-election frequently
if self.hasReelectionQuorum(instId):
logger.debug("{} achieved reelection quorum".format(replica),
extra={"cli": True})
# Need to find the most frequent tie reported to avoid `tie`s
# from malicious nodes. Since lists are not hashable so
# converting each tie(a list of node names) to a tuple.
ties = [
tuple(t)
for t in self.reElectionProposals[instId].values()
]
tieAmong, freq = mostCommonElement(ties)
self.setElectionDefaults(instId)
if not self.hasPrimaryReplica and not self.was_master_primary_in_prev_view:
# There was a tie among this and some other node(s), so do a
# random wait
if replica.name in [_[0] for _ in tieAmong]:
# Try to nominate self after a random delay but dont block
# until that delay and because a nominate from another
# node might be sent
self._schedule(partial(self.nominateReplica, instId),
random.randint(1, 3))
else:
# Now try to nominate self again as there is a
# reelection
self.nominateReplica(instId)
else:
logger.debug("{} does not have re-election quorum yet. "
"Got only {}".format(
replica,
len(self.reElectionProposals[instId])))
else:
self.discard(
reelection,
"already got re-election proposal from {}".format(sndrRep),
logger.debug)
def processReelection(self, reelection: Reelection, sender: str):
"""
Process reelection requests sent by other nodes.
If quorum is achieved, proceed with the reelection process.
:param reelection: the reelection request
:param sender: name of the node from which the reelection was sent
"""
logger.debug(
"{}'s elector started processing reelection msg".format(self.name))
# Check for election round number to discard any previous
# reelection round message
instId = reelection.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, reelection.instId)
if instId not in self.reElectionProposals:
self.setDefaults(instId)
expectedRoundDiff = 0 if (replica.name in
self.reElectionProposals[instId]) else 1
expectedRound = self.reElectionRounds[instId] + expectedRoundDiff
if not reelection.round == expectedRound:
self.discard(reelection,
"reelection request from {} with round "
"number {} does not match expected {}".
format(sndrRep, reelection.round, expectedRound),
logger.debug)
return
if sndrRep not in self.reElectionProposals[instId]:
self.reElectionProposals[instId][sndrRep] = reelection.tieAmong
# Check if got reelection messages from at least 2f + 1 nodes (1
# more than max faulty nodes). Necessary because some nodes may
# turn out to be malicious and send re-election frequently
if self.hasReelectionQuorum(instId):
logger.debug("{} achieved reelection quorum".format(replica),
extra={"cli": True})
# Need to find the most frequent tie reported to avoid `tie`s
# from malicious nodes. Since lists are not hashable so
# converting each tie(a list of node names) to a tuple.
ties = [tuple(t) for t in
self.reElectionProposals[instId].values()]
tieAmong = mostCommonElement(ties)
self.setElectionDefaults(instId)
# There was a tie among this and some other node(s), so do a
# random wait
if replica.name in tieAmong:
# Try to nominate self after a random delay but dont block
# until that delay and because a nominate from another
# node might be sent
self._schedule(partial(self.nominateReplica, instId),
random.randint(1, 3))
else:
# Now try to nominate self again as there is a reelection
self.nominateReplica(instId)
else:
logger.debug(
"{} does not have re-election quorum yet. Got only {}".format(
replica, len(self.reElectionProposals[instId])))
else:
self.discard(reelection,
"already got re-election proposal from {}".
format(sndrRep),
logger.warning)
def processPrimary(self, prim: Primary, sender: str) -> None:
"""
Process a vote from a replica to select a particular replica as primary.
Once 2f + 1 primary declarations have been received, decide on a primary replica.
:param prim: a vote
:param sender: the name of the node from which this message was sent
"""
logger.debug("{}'s elector started processing primary msg from {} : {}"
.format(self.name, sender, prim))
instId = prim.instId
replica = self.replicas[instId]
sndrRep = replica.generateName(sender, prim.instId)
# Nodes should not be able to declare `Primary` winner more than more
if instId not in self.primaryDeclarations:
self.setDefaults(instId)
if sndrRep not in self.primaryDeclarations[instId]:
self.primaryDeclarations[instId][sndrRep] = prim.name
# If got more than 2f+1 primary declarations then in a position to
# decide whether it is the primary or not `2f + 1` declarations
# are enough because even when all the `f` malicious nodes declare
# a primary, we still have f+1 primary declarations from
# non-malicious nodes. One more assumption is that all the non
# malicious nodes vote for the the same primary
# Find for which node there are maximum primary declarations.
# Cant be a tie among 2 nodes since all the non malicious nodes
# which would be greater than or equal to f+1 would vote for the
# same node
if replica.isPrimary is not None:
logger.debug(
"{} Primary already selected; ignoring PRIMARY msg".format(
replica))
return
if self.hasPrimaryQuorum(instId):
if replica.isPrimary is None:
primary = mostCommonElement(
self.primaryDeclarations[instId].values())
logger.display("{} selected primary {} for instance {} "
"(view {})".
format(replica, primary, instId, self.viewNo),
extra={"cli": "ANNOUNCE"})
logger.debug("{} selected primary on the basis of {}".
format(replica,
self.primaryDeclarations[instId]),
extra={"cli": False})
# If the maximum primary declarations are for this node
# then make it primary
replica.primaryName = primary
# If this replica has nominated itself and since the
# election is over, reset the flag
if self.replicaNominatedForItself == instId:
self.replicaNominatedForItself = None
self.node.primaryFound()
self.scheduleElection()
else:
self.discard(prim,
"it already decided primary which is {}".
format(replica.primaryName),
logger.debug)
else:
logger.debug(
"{} received {} but does it not have primary quorum yet"
.format(self.name, prim))
else:
self.discard(prim,
"already got primary declaration from {}".
format(sndrRep),
logger.warning)
key = (Primary.typename, instId, sndrRep)
self.duplicateMsgs[key] = self.duplicateMsgs.get(key, 0) + 1