def __init__(self,
name: str,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.graphStore = self.getGraphStore()
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auto=AutoMode.always)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = SimpleStack(stackargs,
msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set(
) # makes it easier to guard against duplicates
def view_changer(tconf):
node = FakeSomething(
config=tconf,
master_replica=FakeSomething(
inBox=deque(),
inBoxRouter=Router(),
logger=FakeSomething(info=lambda *args, **kwargs: True)),
name="Alpha",
master_primary_name="Alpha",
on_view_change_start=lambda *args, **kwargs: True,
start_catchup=lambda *args, **kwargs: True,
nodeInBox=deque(),
nodeMsgRouter=Router(),
metrics=None,
process_one_node_message=None,
quota_control=FakeSomething(node_quota=Quota(count=100, size=100)),
nodestack=FakeSomething(
service=lambda *args, **kwargs: eventually(lambda: True)))
node.metrics = functools.partial(Node._createMetricsCollector, node)()
node.process_one_node_message = functools.partial(
Node.process_one_node_message, node)
view_changer = ViewChanger(node)
node.view_changer = view_changer
node.viewNo = view_changer.view_no
node.master_replica.node = node
return view_changer
def __init__(self,
name: str = None,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(
stackargs, msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
# To let client send this transactions to just one node
self._read_only_requests = {
GET_NYM, GET_ATTR, GET_CLAIM_DEF, GET_SCHEMA
}
def fake_node(tconf):
node = FakeSomething(config=tconf,
timer=QueueTimer(),
nodeStatusDB=None,
master_replica=FakeSomething(inBox=deque(),
inBoxRouter=Router(),
_external_bus=MockNetwork(),
internal_bus=InternalBus(),
logger=FakeSomething(
info=lambda *args, **kwargs: True
)),
name="Alpha",
master_primary_name="Alpha",
on_view_change_start=lambda *args, **kwargs: True,
start_catchup=lambda *args, **kwargs: True,
nodeInBox=deque(),
nodeMsgRouter=Router(),
metrics=None,
process_one_node_message=None,
quota_control=FakeSomething(
node_quota=Quota(count=100,
size=100)),
nodestack=FakeSomething(
service=lambda *args, **kwargs: eventually(lambda: True)),
set_view_for_replicas= lambda view_no: None,
set_view_change_status=lambda view_no: None
)
node.metrics = functools.partial(Node._createMetricsCollector, node)()
node.process_one_node_message = functools.partial(Node.process_one_node_message, node)
return node
def __init__(self, provider: ViewChangerDataProvider, timer: TimerService):
self.provider = provider
self._timer = timer
self.pre_vc_strategy = None
self._view_no = 0 # type: int
self.inBox = deque()
self.outBox = deque()
self.inBoxRouter = Router(
(InstanceChange, self.process_instance_change_msg),
(ViewChangeDone, self.process_vchd_msg),
(FutureViewChangeDone, self.process_future_view_vchd_msg))
self.instance_changes = InstanceChangeProvider(
self.config.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL,
node_status_db=self.provider.node_status_db)
# Tracks if other nodes are indicating that this node is in lower view
# than others. Keeps a map of view no to senders
# TODO: Consider if sufficient ViewChangeDone for 2 different (and
# higher views) are received, should one view change be interrupted in
# between.
self._next_view_indications = {}
self._view_change_in_progress = False
self.pre_view_change_in_progress = False
self.previous_view_no = None
self.previous_master_primary = None
self.set_defaults()
self.initInsChngThrottling()
# Action for _schedule instanceChange messages
self.instance_change_action = None
# Count of instance change rounds
self.instance_change_rounds = 0
# Time for view_change_starting
self.start_view_change_ts = 0
# Last successful viewNo.
# In some cases view_change process can be uncompleted in time.
# In that case we want to know, which viewNo was successful (last completed view_change)
self.last_completed_view_no = 0
# Force periodic view change if enabled in config
force_view_change_freq = self.config.ForceViewChangeFreq
if force_view_change_freq > 0:
RepeatingTimer(self._timer, force_view_change_freq,
self.on_master_degradation)
# Start periodic freshness check
state_freshness_update_interval = self.config.STATE_FRESHNESS_UPDATE_INTERVAL
if state_freshness_update_interval > 0:
RepeatingTimer(self._timer, state_freshness_update_interval,
self.check_freshness)
def __init__(self, provider: ViewChangerDataProvider, timer: TimerService):
self.provider = provider
self._timer = timer
self.inBox = deque()
self.outBox = deque()
self.inBoxRouter = Router(
(InstanceChange, self.process_instance_change_msg)
)
self.instance_changes = InstanceChangeProvider(self.config.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL,
node_status_db=self.provider.node_status_db)
self.previous_view_no = None
# Action for _schedule instanceChange messages
self.instance_change_action = None
# Count of instance change rounds
self.instance_change_rounds = 0
# Time for view_change_starting
self.start_view_change_ts = 0
# Force periodic view change if enabled in config
force_view_change_freq = self.config.ForceViewChangeFreq
if force_view_change_freq > 0:
RepeatingTimer(self._timer, force_view_change_freq, self.on_master_degradation)
# Start periodic freshness check
state_freshness_update_interval = self.config.STATE_FRESHNESS_UPDATE_INTERVAL
if state_freshness_update_interval > 0:
RepeatingTimer(self._timer, state_freshness_update_interval, self.check_freshness)
def __init__(self, node):
super().__init__(node)
# Flag variable which indicates which replica has nominated for itself
self.replicaNominatedForItself = None
self.nominations = {}
self.primaryDeclarations = {}
self.scheduledPrimaryDecisions = {}
self.reElectionProposals = {}
self.reElectionRounds = {}
# # Tracks when election started for each instance, once
# # `MaxElectionTimeoutFactor`*node_count elapses and no primary decided,
# # re-start election
# self.election_start_times = {}
routerArgs = [(Nomination, self.processNominate),
(Primary, self.processPrimary),
(Reelection, self.processReelection)]
self.inBoxRouter = Router(*routerArgs)
# Keeps track of duplicate messages received. Used to blacklist if
# nodes send more than 1 duplicate messages. Useful to blacklist
# nodes. This number `1` is configurable. The reason 1 duplicate
# message is tolerated is because sometimes when a node communicates
# to an already lagged node, an extra NOMINATE or PRIMARY might be sent
self.duplicateMsgs = {} # Dict[Tuple, int]
def __init__(self,
name: str=None,
nodeReg: Dict[str, HA]=None,
ha: Union[HA, Tuple[str, int]]=None,
peerHA: Union[HA, Tuple[str, int]]=None,
basedirpath: str=None,
config=None,
sighex: str=None):
config = config or getConfig()
super().__init__(name,
nodeReg,
ha,
basedirpath,
config,
sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(stackargs,
msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set() # makes it easier to guard against duplicates
def __init__(self, node):
super().__init__(node)
# TODO: How does primary decider ensure that a node does not have a
# primary while its catching up
self.node = node
self.replicaNominatedForItself = None
"""Flag variable which indicates which replica has nominated
for itself"""
self.nominations = {}
self.primaryDeclarations = {}
self.scheduledPrimaryDecisions = {}
self.reElectionProposals = {}
self.reElectionRounds = {}
routerArgs = [(Nomination, self.processNominate),
(Primary, self.processPrimary),
(Reelection, self.processReelection)]
self.inBoxRouter = Router(*routerArgs)
self.pendingMsgsForViews = {} # Dict[int, deque]
# Keeps track of duplicate messages received. Used to blacklist if
# nodes send more than 1 duplicate messages. Useful to blacklist
# nodes. This number `1` is configurable. The reason 1 duplicate
# message is tolerated is because sometimes when a node communicates
# to an already lagged node, an extra NOMINATE or PRIMARY might be sent
self.duplicateMsgs = {} # Dict[Tuple, int]
def __init__(self, node) -> None:
self._node = node
super().__init__([ObserverSyncPolicyEachBatch(self._node)])
HasActionQueue.__init__(self)
self._inbox = deque()
self._inbox_router = Router((ObservedData, self.apply_data), )
def __init__(self, node):
self.node = node
self._view_no = 0 # type: int
HasActionQueue.__init__(self)
self.inBox = deque()
self.outBox = deque()
self.inBoxRouter = Router(
(InstanceChange, self.process_instance_change_msg),
(ViewChangeDone, self.process_vchd_msg),
(FutureViewChangeDone, self.process_future_view_vchd_msg))
self.instanceChanges = InstanceChanges()
# The quorum of `ViewChangeDone` msgs is different depending on whether we're doing a real view change,
# or just propagating view_no and Primary from `CurrentState` messages sent to a newly joined Node.
# TODO: separate real view change and Propagation of Primary
# TODO: separate catch-up, view-change and primary selection so that
# they are really independent.
self.propagate_primary = False
# Tracks if other nodes are indicating that this node is in lower view
# than others. Keeps a map of view no to senders
# TODO: Consider if sufficient ViewChangeDone for 2 different (and
# higher views) are received, should one view change be interrupted in
# between.
self._next_view_indications = {}
self._view_change_in_progress = False
self.previous_master_primary = None
self.set_defaults()
self.initInsChngThrottling()
# Action for _schedule instanceChange messages
self.instance_change_action = None
# Count of instance change rounds
self.instance_change_rounds = 0
# Time for view_change_starting
self.start_view_change_ts = 0
# Last successful viewNo.
# In some cases view_change process can be uncompleted in time.
# In that case we want to know, which viewNo was successful (last completed view_change)
self.last_completed_view_no = 0
# Force periodic view change if enabled in config
force_view_change_freq = node.config.ForceViewChangeFreq
if force_view_change_freq > 0:
self.startRepeating(self.on_master_degradation,
force_view_change_freq)
def __init__(self) -> None:
HasActionQueue.__init__(self)
self._inbox = deque()
self._outbox = deque()
self._inbox_router = Router((BatchCommitted, self.process_new_batch), )
# TODO: support other policies
self.__sync_policies = {
ObserverSyncPolicyType.EACH_BATCH:
ObservableSyncPolicyEachBatch(self)
}
def __init__(self, node):
HasActionQueue.__init__(self)
self.node = node
self.name = node.name
self.replicas = node.replicas
self.nodeCount = 0
self.inBox = deque()
self.outBox = deque()
self.inBoxRouter = Router(*self.routes)
# Need to keep track of who was primary for the master protocol
# instance for previous view, this variable only matters between
# elections, the elector will set it before doing triggering new
# election and will reset it after primary is decided for master
# instance
self.previous_master_primary = None
def __init__(self, node):
self.node = node
self.view_no = 0 # type: int
HasActionQueue.__init__(self)
self.inBox = deque()
self.outBox = deque()
self.inBoxRouter = Router(
(InstanceChange, self.process_instance_change_msg),
(ViewChangeDone, self.process_vchd_msg)
)
self.instanceChanges = InstanceChanges()
# The quorum of `ViewChangeDone` msgs is different depending on whether we're doing a real view change,
# or just propagating view_no and Primary from `CurrentState` messages sent to a newly joined Node.
# TODO: separate real view change and Propagation of Primary
# TODO: separate catch-up, view-change and primary selection so that
# they are really independent.
self.propagate_primary = False
# Tracks if other nodes are indicating that this node is in lower view
# than others. Keeps a map of view no to senders
# TODO: Consider if sufficient ViewChangeDone for 2 different (and
# higher views) are received, should one view change be interrupted in
# between.
self._next_view_indications = SortedDict()
self._view_change_in_progress = False
self.previous_master_primary = None
self.set_defaults()
self.initInsChngThrottling()
# Action for _schedule instanceChange messages
self.instance_change_action = None
# Count of instance change rounds
self.instance_change_rounds = 0
def __init__(self,
name: str=None,
nodeReg: Dict[str, HA]=None,
ha: Union[HA, Tuple[str, int]]=None,
peerHA: Union[HA, Tuple[str, int]]=None,
basedirpath: str=None,
config=None,
sighex: str=None):
self.config = config or getConfig()
self.setupAnoncreds()
basedirpath = basedirpath or os.path.join(self.config.CLI_NETWORK_DIR, self.config.NETWORK_NAME)
super().__init__(name,
nodeReg,
ha,
basedirpath,
config=config,
sighex=sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(
stackargs, msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
# To let client send this transactions to just one node
self._read_only_requests = {GET_NYM,
GET_ATTR,
GET_CLAIM_DEF,
GET_SCHEMA}
class Client(PlenumClient):
def __init__(self,
name: str = None,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(
stackargs, msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
# To let client send this transactions to just one node
self._read_only_requests = {
GET_NYM, GET_ATTR, GET_CLAIM_DEF, GET_SCHEMA
}
@property
def peerStackClass(self):
if config.UseZStack:
return SimpleZStack
return SimpleRStack
def handlePeerMessage(self, msg):
"""
Use the peerMsgRouter to pass the messages to the correct
function that handles them
:param msg: the P2P client message.
"""
return self.peerMsgRouter.handle(msg)
def getReqRepStore(self):
return ClientReqRepStoreFile(self.name, self.basedirpath)
def getTxnLogStore(self):
return ClientTxnLog(self.name, self.basedirpath)
def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None:
msg, sender = wrappedMsg
# excludeGetTxns = (msg.get(OP_FIELD_NAME) == REPLY and
# msg[f.RESULT.nm].get(TXN_TYPE) == GET_TXNS)
excludeReqAcks = msg.get(OP_FIELD_NAME) == REQACK
excludeReqNacks = msg.get(OP_FIELD_NAME) == REQNACK
excludeReply = msg.get(OP_FIELD_NAME) == REPLY
excludeReject = msg.get(OP_FIELD_NAME) == REJECT
excludeFromCli = excludeFromCli or excludeReqAcks or excludeReqNacks \
or excludeReply or excludeReject
super().handleOneNodeMsg(wrappedMsg, excludeFromCli)
if OP_FIELD_NAME not in msg:
logger.error("Op absent in message {}".format(msg))
def requestConfirmed(self, identifier: str, reqId: int) -> bool:
return self.txnLog.hasTxnWithReqId(identifier, reqId)
def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
return super().hasConsensus(identifier, reqId)
def prepare_for_state(self, result):
request_type = result[TYPE]
if request_type == GET_NYM:
return domain.prepare_get_nym_for_state(result)
if request_type == GET_ATTR:
path, value, hashed_value, value_bytes = \
domain.prepare_get_attr_for_state(result)
return path, value_bytes
if request_type == GET_CLAIM_DEF:
return domain.prepare_get_claim_def_for_state(result)
if request_type == GET_SCHEMA:
return domain.prepare_get_schema_for_state(result)
raise ValueError("Cannot make state key for "
"request of type {}".format(request_type))
def getTxnsByType(self, txnType):
return self.txnLog.getTxnsByType(txnType)
# TODO: Just for now. Remove it later
def doAttrDisclose(self, origin, target, txnId, key):
box = libnacl.public.Box(b58decode(origin), b58decode(target))
data = json.dumps({TXN_ID: txnId, SKEY: key})
nonce, boxedMsg = box.encrypt(data.encode(), pack_nonce=False)
op = {
TARGET_NYM: target,
TXN_TYPE: DISCLO,
NONCE: b58encode(nonce),
DATA: b58encode(boxedMsg)
}
self.submit(op, identifier=origin)
def doGetAttributeTxn(self, identifier, attrName):
op = {
TARGET_NYM: identifier,
TXN_TYPE: GET_ATTR,
DATA: json.dumps({"name": attrName})
}
self.submit(op, identifier=identifier)
@staticmethod
def _getDecryptedData(encData, key):
data = bytes(bytearray.fromhex(encData))
rawKey = bytes(bytearray.fromhex(key))
box = libnacl.secret.SecretBox(rawKey)
decData = box.decrypt(data).decode()
return json.loads(decData)
def hasNym(self, nym):
for txn in self.txnLog.getTxnsByType(NYM):
if txn.get(TXN_TYPE) == NYM:
return True
return False
def _statusChanged(self, old, new):
super()._statusChanged(old, new)
def start(self, loop):
super().start(loop)
if self.hasAnonCreds and self.status not in Status.going():
self.peerStack.start()
async def prod(self, limit) -> int:
s = await super().prod(limit)
if self.hasAnonCreds:
return s + await self.peerStack.service(limit)
else:
return s
class Replica(HasActionQueue, MessageProcessor):
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
@property
def h(self) -> int:
return self._h
@h.setter
def h(self, n):
self._h = n
self.H = self._h + self.config.LOG_SIZE
@property
def requests(self):
return self.node.requests
def shouldParticipate(self, viewNo: int, ppSeqNo: int):
# Replica should only participating in the consensus process and the
# replica did not stash any of this request's 3-phase request
return self.node.isParticipating and (viewNo, ppSeqNo) \
not in self.stashingWhileCatchingUp
@staticmethod
def generateName(nodeName: str, instId: int):
"""
Create and return the name for a replica using its nodeName and
instanceId.
Ex: Alpha:1
"""
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".
format(self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".
format(self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicion exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicion exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo == self.viewNo
def isMsgForPrevView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
if self.isMsgForLaterView(msg):
self.discard(msg,
"Cannot get primary status for a request for a later "
"view {}. Request is {}".format(self.viewNo, msg),
logger.error)
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
logger.error("{} cannot get primary for a request for a later "
"view. Request is {}".format(self, msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
logger.debug("{} stashing request digest {} since it does not know "
"its primary status".
format(self, (rd.identifier, rd.reqId)))
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
r = self.inBoxRouter.handleAllSync(self.inBox, limit)
r += self._serviceActions()
return r
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}"
.format(self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param msg: the ThreePhaseMsg to dispatch
:param sender: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
if self.isPpSeqNoAcceptable(msg.ppSeqNo):
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
else:
logger.debug("{} stashing 3 phase message {} since ppSeqNo {} is "
"not between {} and {}".
format(self, msg, msg.ppSeqNo, self.h, self.H))
self.stashingWhileOutsideWaterMarks.append((msg, sender))
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
self.dequeuePrePrepare(rd.identifier, rd.reqId)
else:
self.doPrePrepare(rd)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug("{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".
format(self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
key = (pp.viewNo, pp.ppSeqNo)
logger.debug("{} Receiving PRE-PREPARE{} at {} from {}".
format(self, key, time.perf_counter(), sender))
if self.canProcessPrePrepare(pp, sender):
if not self.node.isParticipating:
self.stashingWhileCatchingUp.add(key)
self.addToPrePrepares(pp)
logger.info("{} processed incoming PRE-PREPARE{}".
format(self, key))
def tryPrepare(self, pp: PrePrepare):
"""
Try to send the Prepare message if the PrePrepare message is ready to
be passed into the Prepare phase.
"""
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
logger.debug("{} received PREPARE{} from {}".
format(self, (prepare.viewNo, prepare.ppSeqNo), sender))
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
logger.debug("{} processed incoming PREPARE {}".
format(self, (prepare.viewNo, prepare.ppSeqNo)))
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning("{} cannot process incoming PREPARE".
format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received COMMIT {} from {}".
format(self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
logger.debug("{} processed incoming COMMIT{}".
format(self, (commit.viewNo, commit.ppSeqNo)))
def tryCommit(self, prepare: Prepare):
"""
Try to commit if the Prepare message is ready to be passed into the
commit phase.
"""
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
"""
Try to order if the Commit message is ready to be ordered.
"""
canOrder, reason = self.canOrder(commit)
if canOrder:
logger.debug("{} returning request to node".format(self))
self.tryOrdering(commit)
else:
logger.trace("{} cannot return request to node: {}".
format(self, reason))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements identifier, reqId, and digest
"""
if not self.node.isParticipating:
logger.error("Non participating node is attempting PRE-PREPARE. "
"This should not happen.")
return
if self.lastPrePrepareSeqNo == self.H:
logger.debug("{} stashing PRE-PREPARE {} since outside greater "
"than high water mark {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo+1),
self.H))
self.stashingWhileOutsideWaterMarks.append(reqDigest)
return
self.lastPrePrepareSeqNo += 1
tm = time.time()*1000
logger.debug("{} Sending PRE-PREPARE {} at {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo),
time.perf_counter()))
prePrepareReq = PrePrepare(self.instId,
self.viewNo,
self.lastPrePrepareSeqNo,
*reqDigest,
tm)
self.sentPrePrepares[self.viewNo, self.lastPrePrepareSeqNo] = (reqDigest.key,
tm)
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE {} at {}".
format(self, (pp.viewNo, pp.ppSeqNo), time.perf_counter()))
prepare = Prepare(self.instId,
pp.viewNo,
pp.ppSeqNo,
pp.digest,
pp.ppTime)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
"""
Create a commit message from the given Prepare message and trigger the
commit phase
:param p: the prepare message
"""
logger.debug("{} Sending COMMIT{} at {}".
format(self, (p.viewNo, p.ppSeqNo), time.perf_counter()))
commit = Commit(self.instId,
p.viewNo,
p.ppSeqNo,
p.digest,
p.ppTime)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool:
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# TODO: Check whether it is rejecting PRE-PREPARE from previous view
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
# A PRE-PREPARE is sent that has already been received
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
key = (pp.identifier, pp.reqId)
if not self.requests.isFinalised(key):
self.enqueuePrePrepare(pp, sender)
return False
# A PRE-PREPARE is sent that does not match request digest
if self.requests.digest(key) != pp.digest:
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
key = (pp.viewNo, pp.ppSeqNo)
self.prePrepares[key] = \
((pp.identifier, pp.reqId), pp.ppTime)
self.dequeuePrepares(*key)
self.dequeueCommits(*key)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def hasPrepared(self, request) -> bool:
return self.prepares.hasPrepareFrom(request, self.name)
def canSendPrepare(self, request) -> bool:
"""
Return whether the request identified by (identifier, requestId) can
proceed to the Prepare step.
:param request: any object with identifier and requestId attributes
"""
return self.shouldParticipate(request.viewNo, request.ppSeqNo) \
and not self.hasPrepared(request) \
and self.requests.isFinalised((request.identifier,
request.reqId))
def isValidPrepare(self, prepare: Prepare, sender: str) -> bool:
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
return False
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT, prepare)
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def hasCommitted(self, request) -> bool:
return self.commits.hasCommitFrom(ThreePhaseKey(
request.viewNo, request.ppSeqNo), self.name)
def canCommit(self, prepare: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param prepare: the PREPARE
"""
return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \
self.prepares.hasQuorum(prepare, self.f) and \
not self.hasCommitted(prepare)
def isValidCommit(self, commit: Commit, sender: str) -> bool:
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
primaryStatus = self.isPrimaryForMsg(commit)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
key = (commit.viewNo, commit.ppSeqNo)
if key not in ppReqs:
self.enqueueCommit(commit, sender)
return False
if (key not in self.prepares and
key not in self.preparesWaitingForPrePrepare):
logger.debug("{} rejecting COMMIT{} due to lack of prepares".
format(self, key))
# raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
return False
elif self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
elif commit.digest != self.getDigestFor3PhaseKey(ThreePhaseKey(*key)):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
elif key in ppReqs and commit.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG,
commit)
else:
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def hasOrdered(self, viewNo, ppSeqNo) -> bool:
return (viewNo, ppSeqNo) in self.ordered
def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
if not self.commits.hasQuorum(commit, self.f):
return False, "no quorum: {} commits where f is {}".\
format(commit, self.f)
if self.hasOrdered(commit.viewNo, commit.ppSeqNo):
return False, "already ordered"
if not self.isNextInOrdering(commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if viewNo not in self.stashedCommitsForOrdering:
self.stashedCommitsForOrdering[viewNo] = {}
self.stashedCommitsForOrdering[viewNo][ppSeqNo] = commit
# self._schedule(self.orderStashedCommits, 2)
self.startRepeating(self.orderStashedCommits, 2)
return False, "stashing {} since out of order".\
format(commit)
return True, None
def isNextInOrdering(self, commit: Commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if self.ordered and self.ordered[-1] == (viewNo, ppSeqNo-1):
return True
for (v, p) in self.commits:
if v < viewNo:
# Have commits from previous view that are unordered.
# TODO: Question: would commits be always ordered, what if
# some are never ordered and its fine, go to PBFT.
return False
if v == viewNo and p < ppSeqNo and (v, p) not in self.ordered:
# If unordered commits are found with lower ppSeqNo then this
# cannot be ordered.
return False
# TODO: Revisit PBFT paper, how to make sure that last request of the
# last view has been ordered? Need change in `VIEW CHANGE` mechanism.
# Somehow view change needs to communicate what the last request was.
# Also what if some COMMITs were completely missed in the same view
return True
def orderStashedCommits(self):
# TODO: What if the first few commits were out of order and stashed?
# `self.ordered` would be empty
if self.ordered:
lastOrdered = self.ordered[-1]
vToRemove = set()
for v in self.stashedCommitsForOrdering:
if v < lastOrdered[0] and self.stashedCommitsForOrdering[v]:
raise RuntimeError("{} found commits from previous view {}"
" that were not ordered but last ordered"
" is {}".format(self, v, lastOrdered))
pToRemove = set()
for p, commit in self.stashedCommitsForOrdering[v].items():
if (v == lastOrdered[0] and lastOrdered == (v, p - 1)) or \
(v > lastOrdered[0] and
self.isLowestCommitInView(commit)):
logger.debug("{} ordering stashed commit {}".
format(self, commit))
if self.tryOrdering(commit):
lastOrdered = (v, p)
pToRemove.add(p)
for p in pToRemove:
del self.stashedCommitsForOrdering[v][p]
if not self.stashedCommitsForOrdering[v]:
vToRemove.add(v)
for v in vToRemove:
del self.stashedCommitsForOrdering[v]
# if self.stashedCommitsForOrdering:
# self._schedule(self.orderStashedCommits, 2)
if not self.stashedCommitsForOrdering:
self.stopRepeating(self.orderStashedCommits)
def isLowestCommitInView(self, commit):
# TODO: Assumption: This assumes that at least one commit that was sent
# for any request by any node has been received in the view of this
# commit
ppSeqNos = []
for v, p in self.commits:
if v == commit.viewNo:
ppSeqNos.append(p)
return min(ppSeqNos) == commit.ppSeqNo if ppSeqNos else True
def tryOrdering(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
logger.debug("{} trying to order COMMIT{}".format(self, key))
reqKey = self.getReqKeyFrom3PhaseKey(key) # type: Tuple
digest = self.getDigestFor3PhaseKey(key)
if not digest:
logger.error("{} did not find digest for {}, request key {}".
format(self, key, reqKey))
return
self.doOrder(*key, *reqKey, digest, commit.ppTime)
return True
def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime):
key = (viewNo, ppSeqNo)
self.addToOrdered(*key)
ordered = Ordered(self.instId,
viewNo,
identifier,
reqId,
ppTime)
# TODO: Should not order or add to checkpoint while syncing
# 3 phase state.
self.send(ordered, TPCStat.OrderSent)
if key in self.stashingWhileCatchingUp:
self.stashingWhileCatchingUp.remove(key)
logger.debug("{} ordered request {}".format(self, (viewNo, ppSeqNo)))
self.addToCheckpoint(ppSeqNo, digest)
def processCheckpoint(self, msg: Checkpoint, sender: str):
if self.checkpoints:
seqNo = msg.seqNo
_, firstChk = self.firstCheckPoint
if firstChk.isStable:
if firstChk.seqNo == seqNo:
self.discard(msg, reason="Checkpoint already stable",
logMethod=logger.debug)
return
if firstChk.seqNo > seqNo:
self.discard(msg, reason="Higher stable checkpoint present",
logMethod=logger.debug)
return
for state in self.checkpoints.values():
if state.seqNo == seqNo:
if state.digest == msg.digest:
state.receivedDigests[sender] = msg.digest
break
else:
logger.error("{} received an incorrect digest {} for "
"checkpoint {} from {}".format(self,
msg.digest,
seqNo,
sender))
return
if len(state.receivedDigests) == 2*self.f:
self.markCheckPointStable(msg.seqNo)
else:
self.discard(msg, reason="No checkpoints present to tally",
logMethod=logger.warn)
def _newCheckpointState(self, ppSeqNo, digest) -> CheckpointState:
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ - 1
logger.debug("{} adding new checkpoint state for {}".
format(self, (s, e)))
state = CheckpointState(ppSeqNo, [digest, ], None, {}, False)
self.checkpoints[s, e] = state
return state
def addToCheckpoint(self, ppSeqNo, digest):
for (s, e) in self.checkpoints.keys():
if s <= ppSeqNo <= e:
state = self.checkpoints[s, e] # type: CheckpointState
state.digests.append(digest)
state = updateNamedTuple(state, seqNo=ppSeqNo)
self.checkpoints[s, e] = state
break
else:
state = self._newCheckpointState(ppSeqNo, digest)
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ
if len(state.digests) == self.config.CHK_FREQ:
state = updateNamedTuple(state, digest=serialize(state.digests),
digests=[])
self.checkpoints[s, e] = state
self.send(Checkpoint(self.instId, self.viewNo, ppSeqNo,
state.digest))
def markCheckPointStable(self, seqNo):
previousCheckpoints = []
for (s, e), state in self.checkpoints.items():
if e == seqNo:
state = updateNamedTuple(state, isStable=True)
self.checkpoints[s, e] = state
break
else:
previousCheckpoints.append((s, e))
else:
logger.error("{} could not find {} in checkpoints".
format(self, seqNo))
return
self.h = seqNo
for k in previousCheckpoints:
logger.debug("{} removing previous checkpoint {}".format(self, k))
self.checkpoints.pop(k)
self.gc(seqNo)
logger.debug("{} marked stable checkpoint {}".format(self, (s, e)))
self.processStashedMsgsForNewWaterMarks()
def gc(self, tillSeqNo):
logger.debug("{} cleaning up till {}".format(self, tillSeqNo))
tpcKeys = set()
reqKeys = set()
for (v, p), (reqKey, _) in self.sentPrePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
for (v, p), (reqKey, _) in self.prePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
logger.debug("{} found {} 3 phase keys to clean".
format(self, len(tpcKeys)))
logger.debug("{} found {} request keys to clean".
format(self, len(reqKeys)))
for k in tpcKeys:
self.sentPrePrepares.pop(k, None)
self.prePrepares.pop(k, None)
self.prepares.pop(k, None)
self.commits.pop(k, None)
if k in self.ordered:
self.ordered.remove(k)
for k in reqKeys:
self.requests.pop(k, None)
def processStashedMsgsForNewWaterMarks(self):
while self.stashingWhileOutsideWaterMarks:
item = self.stashingWhileOutsideWaterMarks.pop()
logger.debug("{} processing stashed item {} after new stable "
"checkpoint".format(self, item))
if isinstance(item, ReqDigest):
self.doPrePrepare(item)
elif isinstance(item, tuple) and len(tuple) == 2:
self.dispatchThreePhaseMsg(*item)
else:
logger.error("{} cannot process {} "
"from stashingWhileOutsideWaterMarks".
format(self, item))
@property
def firstCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(0)
@property
def lastCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(-1)
def isPpSeqNoAcceptable(self, ppSeqNo: int):
return self.h < ppSeqNo <= self.H
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrePrepare(self, request: PrePrepare, sender: str):
logger.debug("Queueing pre-prepares due to unavailability of finalised "
"Request. Request {} from {}".format(request, sender))
key = (request.identifier, request.reqId)
if key not in self.prePreparesPendingReqDigest:
self.prePreparesPendingReqDigest[key] = []
self.prePreparesPendingReqDigest[key].append((request, sender))
def dequeuePrePrepare(self, identifier: int, reqId: int):
key = (identifier, reqId)
if key in self.prePreparesPendingReqDigest:
pps = self.prePreparesPendingReqDigest[key]
for (pp, sender) in pps:
logger.debug("{} popping stashed PRE-PREPARE{}".
format(self, key))
if pp.digest == self.requests.digest(key):
self.prePreparesPendingReqDigest.pop(key)
self.processPrePrepare(pp, sender)
logger.debug(
"{} processed {} PRE-PREPAREs waiting for finalised "
"request for identifier {} and reqId {}".
format(self, pp, identifier, reqId))
break
def enqueuePrepare(self, request: Prepare, sender: str):
logger.debug("Queueing prepares due to unavailability of PRE-PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
logger.debug("{} popping stashed PREPARE{}".format(self, key))
self.processPrepare(prepare, sender)
i += 1
self.preparesWaitingForPrePrepare.pop(key)
logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def enqueueCommit(self, request: Commit, sender: str):
logger.debug("Queueing commit due to unavailability of PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.commitsWaitingForPrepare:
self.commitsWaitingForPrepare[key] = deque()
self.commitsWaitingForPrepare[key].append((request, sender))
def dequeueCommits(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.commitsWaitingForPrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.commitsWaitingForPrepare[key]:
commit, sender = self.commitsWaitingForPrepare[
key].popleft()
logger.debug("{} popping stashed COMMIT{}".format(self, key))
self.processCommit(commit, sender)
i += 1
self.commitsWaitingForPrepare.pop(key)
logger.debug("{} processed {} COMMITs waiting for PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def getDigestFor3PhaseKey(self, key: ThreePhaseKey) -> Optional[str]:
reqKey = self.getReqKeyFrom3PhaseKey(key)
digest = self.requests.digest(reqKey)
if not digest:
logger.debug("{} could not find digest in sent or received "
"PRE-PREPAREs or PREPAREs for 3 phase key {} and req "
"key {}".format(self, key, reqKey))
return None
else:
return digest
def getReqKeyFrom3PhaseKey(self, key: ThreePhaseKey):
reqKey = None
if key in self.sentPrePrepares:
reqKey = self.sentPrePrepares[key][0]
elif key in self.prePrepares:
reqKey = self.prePrepares[key][0]
elif key in self.prepares:
reqKey = self.prepares[key][0]
else:
logger.debug("Could not find request key for 3 phase key {}".
format(key))
return reqKey
@property
def threePhaseState(self):
# TODO: This method is incomplete
# Gets the current stable and unstable checkpoints and creates digest
# of unstable checkpoints
if self.checkpoints:
pass
else:
state = []
return ThreePCState(self.instId, state)
def process3PhaseState(self, msg: ThreePCState, sender: str):
# TODO: This is not complete
pass
def send(self, msg, stat=None) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.display("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
if stat:
self.stats.inc(stat)
self.outBox.append(msg)
class Client(PlenumClient):
def __init__(self,
name: str,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.graphStore = self.getGraphStore()
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auto=AutoMode.always)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = SimpleStack(stackargs,
msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set(
) # makes it easier to guard against duplicates
def handlePeerMessage(self, msg):
"""
Use the peerMsgRouter to pass the messages to the correct
function that handles them
:param msg: the P2P client message.
"""
return self.peerMsgRouter.handle(msg)
def _getOrientDbStore(self):
return OrientDbStore(user=self.config.OrientDB["user"],
password=self.config.OrientDB["password"],
dbName=self.name,
storageType=pyorient.STORAGE_TYPE_PLOCAL)
def getReqRepStore(self):
if self.config.ReqReplyStore == "orientdb":
return ClientReqRepStoreOrientDB(self._getOrientDbStore())
else:
return ClientReqRepStoreFile(self.name, self.basedirpath)
def getGraphStore(self):
return IdentityGraph(self._getOrientDbStore()) if \
self.config.ClientIdentityGraph else None
def getTxnLogStore(self):
return ClientTxnLog(self.name, self.basedirpath)
def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None:
msg, sender = wrappedMsg
# excludeGetTxns = (msg.get(OP_FIELD_NAME) == REPLY and
# msg[f.RESULT.nm].get(TXN_TYPE) == GET_TXNS)
excludeReqAcks = msg.get(OP_FIELD_NAME) == REQACK
excludeReqNacks = msg.get(OP_FIELD_NAME) == REQNACK
excludeReply = msg.get(OP_FIELD_NAME) == REPLY
excludeFromCli = excludeFromCli or excludeReqAcks or excludeReqNacks \
or excludeReply
super().handleOneNodeMsg(wrappedMsg, excludeFromCli)
if OP_FIELD_NAME not in msg:
logger.error("Op absent in message {}".format(msg))
def postReplyRecvd(self, identifier, reqId, frm, result, numReplies):
reply = super().postReplyRecvd(identifier, reqId, frm, result,
numReplies)
if reply:
for name in self._observers:
try:
self._observers[name](name, reqId, frm, result, numReplies)
except Exception as ex:
# TODO: All errors should not be shown on CLI, or maybe we
# show errors with different color according to the
# severity. Like an error occurring due to node sending
# a malformed message should not result in an error message
# being shown on the cli since the clients would anyway
# collect enough replies from other nodes.
logger.debug("Observer threw an exception", exc_info=ex)
if isinstance(self.reqRepStore, ClientReqRepStoreOrientDB):
self.reqRepStore.setConsensus(identifier, reqId)
if result[TXN_TYPE] == NYM:
if self.graphStore:
self.addNymToGraph(result)
elif result[TXN_TYPE] == ATTRIB:
if self.graphStore:
self.graphStore.addAttribTxnToGraph(result)
elif result[TXN_TYPE] == GET_NYM:
if self.graphStore:
if DATA in result and result[DATA]:
self.addNymToGraph(json.loads(result[DATA]))
elif result[TXN_TYPE] == GET_TXNS:
if DATA in result and result[DATA]:
data = json.loads(result[DATA])
self.reqRepStore.setLastTxnForIdentifier(
result[f.IDENTIFIER.nm], data[LAST_TXN])
if self.graphStore:
for txn in data[TXNS]:
if txn[TXN_TYPE] == NYM:
self.addNymToGraph(txn)
elif txn[TXN_TYPE] == ATTRIB:
try:
self.graphStore.addAttribTxnToGraph(txn)
except pyorient.PyOrientCommandException as ex:
fault(
ex, "An exception was raised while "
"adding attribute")
elif result[TXN_TYPE] == CLAIM_DEF:
if self.graphStore:
self.graphStore.addClaimDefTxnToGraph(result)
elif result[TXN_TYPE] == ISSUER_KEY:
if self.graphStore:
self.graphStore.addIssuerKeyTxnToGraph(result)
# else:
# logger.debug("Unknown type {}".format(result[TXN_TYPE]))
def requestConfirmed(self, identifier: str, reqId: int) -> bool:
if isinstance(self.reqRepStore, ClientReqRepStoreOrientDB):
return self.reqRepStore.requestConfirmed(identifier, reqId)
else:
return self.txnLog.hasTxnWithReqId(identifier, reqId)
def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
if isinstance(self.reqRepStore, ClientReqRepStoreOrientDB):
return self.reqRepStore.hasConsensus(identifier, reqId)
else:
return super().hasConsensus(identifier, reqId)
def addNymToGraph(self, txn):
origin = txn.get(f.IDENTIFIER.nm)
if txn.get(ROLE) == SPONSOR:
if not self.graphStore.hasSteward(origin):
try:
self.graphStore.addNym(None, nym=origin, role=STEWARD)
except pyorient.PyOrientCommandException as ex:
logger.trace("Error occurred adding nym to graph")
logger.trace(traceback.format_exc())
self.graphStore.addNymTxnToGraph(txn)
def getTxnById(self, txnId: str):
if self.graphStore:
txns = list(self.graphStore.getResultForTxnIds(txnId).values())
return txns[0] if txns else {}
else:
# TODO: Add support for fetching reply by transaction id
# serTxn = self.reqRepStore.getResultForTxnId(txnId)
pass
# TODO Add merkleInfo as well
def getTxnsByNym(self, nym: str):
raise NotImplementedError
def getTxnsByType(self, txnType):
if self.graphStore:
edgeClass = getEdgeByTxnType(txnType)
if edgeClass:
cmd = "select from {}".format(edgeClass)
result = self.graphStore.client.command(cmd)
if result:
return [r.oRecordData for r in result]
return []
else:
txns = self.txnLog.getTxnsByType(txnType)
# TODO: Fix ASAP
if txnType == CLAIM_DEF:
for txn in txns:
txn[DATA] = json.loads(txn[DATA].replace(
"\'", '"').replace('"{', '{').replace('}"', '}'))
txn[NAME] = txn[DATA][NAME]
txn[VERSION] = txn[DATA][VERSION]
return txns
# TODO: Just for now. Remove it later
def doAttrDisclose(self, origin, target, txnId, key):
box = libnacl.public.Box(b58decode(origin), b58decode(target))
data = json.dumps({TXN_ID: txnId, SKEY: key})
nonce, boxedMsg = box.encrypt(data.encode(), pack_nonce=False)
op = {
TARGET_NYM: target,
TXN_TYPE: DISCLO,
NONCE: b58encode(nonce),
DATA: b58encode(boxedMsg)
}
self.submit(op, identifier=origin)
def doGetAttributeTxn(self, identifier, attrName):
op = {
TARGET_NYM: identifier,
TXN_TYPE: GET_ATTR,
DATA: json.dumps({"name": attrName})
}
self.submit(op, identifier=identifier)
@staticmethod
def _getDecryptedData(encData, key):
data = bytes(bytearray.fromhex(encData))
rawKey = bytes(bytearray.fromhex(key))
box = libnacl.secret.SecretBox(rawKey)
decData = box.decrypt(data).decode()
return json.loads(decData)
def hasNym(self, nym):
if self.graphStore:
return self.graphStore.hasNym(nym)
else:
for txn in self.txnLog.getTxnsByType(NYM):
if txn.get(TXN_TYPE) == NYM:
return True
return False
def _statusChanged(self, old, new):
super()._statusChanged(old, new)
def start(self, loop):
super().start(loop)
if self.hasAnonCreds and self.status not in Status.going():
self.peerStack.start()
async def prod(self, limit) -> int:
# s = await self.nodestack.service(limit)
# if self.isGoing():
# await self.nodestack.serviceLifecycle()
# self.nodestack.flushOutBoxes()
s = await super().prod(limit)
if self.hasAnonCreds:
return s + await self.peerStack.service(limit)
else:
return s
def registerObserver(self, observer: Callable, name=None):
if not name:
name = uuid.uuid4()
if name in self._observers or observer in self._observerSet:
raise RuntimeError("Observer {} already registered".format(name))
self._observers[name] = observer
self._observerSet.add(observer)
def deregisterObserver(self, name):
if name not in self._observers:
raise RuntimeError("Observer {} not registered".format(name))
self._observerSet.remove(self._observers[name])
del self._observers[name]
def hasObserver(self, name):
return name in self._observerSet
class Client(PlenumClient):
def __init__(self,
name: str=None,
nodeReg: Dict[str, HA]=None,
ha: Union[HA, Tuple[str, int]]=None,
peerHA: Union[HA, Tuple[str, int]]=None,
basedirpath: str=None,
config=None,
sighex: str=None):
config = config or getConfig()
super().__init__(name,
nodeReg,
ha,
basedirpath,
config,
sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(stackargs,
msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set() # makes it easier to guard against duplicates
@property
def peerStackClass(self):
if config.UseZStack:
return SimpleZStack
return SimpleRStack
def handlePeerMessage(self, msg):
"""
Use the peerMsgRouter to pass the messages to the correct
function that handles them
:param msg: the P2P client message.
"""
return self.peerMsgRouter.handle(msg)
def getReqRepStore(self):
return ClientReqRepStoreFile(self.name, self.basedirpath)
def getTxnLogStore(self):
return ClientTxnLog(self.name, self.basedirpath)
def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None:
msg, sender = wrappedMsg
# excludeGetTxns = (msg.get(OP_FIELD_NAME) == REPLY and
# msg[f.RESULT.nm].get(TXN_TYPE) == GET_TXNS)
excludeReqAcks = msg.get(OP_FIELD_NAME) == REQACK
excludeReqNacks = msg.get(OP_FIELD_NAME) == REQNACK
excludeReply = msg.get(OP_FIELD_NAME) == REPLY
excludeReject = msg.get(OP_FIELD_NAME) == REJECT
excludeFromCli = excludeFromCli or excludeReqAcks or excludeReqNacks \
or excludeReply or excludeReject
super().handleOneNodeMsg(wrappedMsg, excludeFromCli)
if OP_FIELD_NAME not in msg:
logger.error("Op absent in message {}".format(msg))
def postReplyRecvd(self, identifier, reqId, frm, result, numReplies):
reply = super().postReplyRecvd(identifier, reqId, frm, result, numReplies)
if reply:
for name in self._observers:
try:
self._observers[name](name, reqId, frm, result, numReplies)
except Exception as ex:
# TODO: All errors should not be shown on CLI, or maybe we
# show errors with different color according to the
# severity. Like an error occurring due to node sending
# a malformed message should not result in an error message
# being shown on the cli since the clients would anyway
# collect enough replies from other nodes.
logger.debug("Observer threw an exception", exc_info=ex)
def requestConfirmed(self, identifier: str, reqId: int) -> bool:
return self.txnLog.hasTxnWithReqId(identifier, reqId)
def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
return super().hasConsensus(identifier, reqId)
def getTxnsByNym(self, nym: str):
raise NotImplementedError
def getTxnsByType(self, txnType):
txns = self.txnLog.getTxnsByType(txnType)
if txnType == SCHEMA:
for txn in txns:
txn[DATA] = json.loads(txn[DATA].replace("\'", '"')
.replace('"{', '{')
.replace('}"', '}'))
txn[NAME] = txn[DATA][NAME]
txn[VERSION] = txn[DATA][VERSION]
return txns
# TODO: Just for now. Remove it later
def doAttrDisclose(self, origin, target, txnId, key):
box = libnacl.public.Box(b58decode(origin), b58decode(target))
data = json.dumps({TXN_ID: txnId, SKEY: key})
nonce, boxedMsg = box.encrypt(data.encode(), pack_nonce=False)
op = {
TARGET_NYM: target,
TXN_TYPE: DISCLO,
NONCE: b58encode(nonce),
DATA: b58encode(boxedMsg)
}
self.submit(op, identifier=origin)
def doGetAttributeTxn(self, identifier, attrName):
op = {
TARGET_NYM: identifier,
TXN_TYPE: GET_ATTR,
DATA: json.dumps({"name": attrName})
}
self.submit(op, identifier=identifier)
@staticmethod
def _getDecryptedData(encData, key):
data = bytes(bytearray.fromhex(encData))
rawKey = bytes(bytearray.fromhex(key))
box = libnacl.secret.SecretBox(rawKey)
decData = box.decrypt(data).decode()
return json.loads(decData)
def hasNym(self, nym):
for txn in self.txnLog.getTxnsByType(NYM):
if txn.get(TXN_TYPE) == NYM:
return True
return False
def _statusChanged(self, old, new):
super()._statusChanged(old, new)
def start(self, loop):
super().start(loop)
if self.hasAnonCreds and self.status not in Status.going():
self.peerStack.start()
async def prod(self, limit) -> int:
s = await super().prod(limit)
if self.hasAnonCreds:
return s + await self.peerStack.service(limit)
else:
return s
def registerObserver(self, observer: Callable, name=None):
if not name:
name = uuid.uuid4()
if name in self._observers or observer in self._observerSet:
raise RuntimeError("Observer {} already registered".format(name))
self._observers[name] = observer
self._observerSet.add(observer)
def deregisterObserver(self, name):
if name not in self._observers:
raise RuntimeError("Observer {} not registered".format(name))
self._observerSet.remove(self._observers[name])
del self._observers[name]
def hasObserver(self, name):
return name in self._observerSet
class Client(PlenumClient):
def __init__(self,
name: str = None,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(
stackargs, msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set(
) # makes it easier to guard against duplicates
@property
def peerStackClass(self):
if config.UseZStack:
return SimpleZStack
return SimpleRStack
def handlePeerMessage(self, msg):
"""
Use the peerMsgRouter to pass the messages to the correct
function that handles them
:param msg: the P2P client message.
"""
return self.peerMsgRouter.handle(msg)
def getReqRepStore(self):
return ClientReqRepStoreFile(self.name, self.basedirpath)
def getTxnLogStore(self):
return ClientTxnLog(self.name, self.basedirpath)
def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None:
msg, sender = wrappedMsg
# excludeGetTxns = (msg.get(OP_FIELD_NAME) == REPLY and
# msg[f.RESULT.nm].get(TXN_TYPE) == GET_TXNS)
excludeReqAcks = msg.get(OP_FIELD_NAME) == REQACK
excludeReqNacks = msg.get(OP_FIELD_NAME) == REQNACK
excludeReply = msg.get(OP_FIELD_NAME) == REPLY
excludeReject = msg.get(OP_FIELD_NAME) == REJECT
excludeFromCli = excludeFromCli or excludeReqAcks or excludeReqNacks \
or excludeReply or excludeReject
super().handleOneNodeMsg(wrappedMsg, excludeFromCli)
if OP_FIELD_NAME not in msg:
logger.error("Op absent in message {}".format(msg))
def postReplyRecvd(self, identifier, reqId, frm, result, numReplies):
reply = super().postReplyRecvd(identifier, reqId, frm, result,
numReplies)
if reply:
for name in self._observers:
try:
self._observers[name](name, reqId, frm, result, numReplies)
except Exception as ex:
# TODO: All errors should not be shown on CLI, or maybe we
# show errors with different color according to the
# severity. Like an error occurring due to node sending
# a malformed message should not result in an error message
# being shown on the cli since the clients would anyway
# collect enough replies from other nodes.
logger.debug("Observer threw an exception", exc_info=ex)
def requestConfirmed(self, identifier: str, reqId: int) -> bool:
return self.txnLog.hasTxnWithReqId(identifier, reqId)
def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
return super().hasConsensus(identifier, reqId)
def getTxnsByNym(self, nym: str):
raise NotImplementedError
def getTxnsByType(self, txnType):
txns = self.txnLog.getTxnsByType(txnType)
if txnType == SCHEMA:
for txn in txns:
txn[DATA] = json.loads(txn[DATA].replace("\'", '"').replace(
'"{', '{').replace('}"', '}'))
txn[NAME] = txn[DATA][NAME]
txn[VERSION] = txn[DATA][VERSION]
return txns
# TODO: Just for now. Remove it later
def doAttrDisclose(self, origin, target, txnId, key):
box = libnacl.public.Box(b58decode(origin), b58decode(target))
data = json.dumps({TXN_ID: txnId, SKEY: key})
nonce, boxedMsg = box.encrypt(data.encode(), pack_nonce=False)
op = {
TARGET_NYM: target,
TXN_TYPE: DISCLO,
NONCE: b58encode(nonce),
DATA: b58encode(boxedMsg)
}
self.submit(op, identifier=origin)
def doGetAttributeTxn(self, identifier, attrName):
op = {
TARGET_NYM: identifier,
TXN_TYPE: GET_ATTR,
DATA: json.dumps({"name": attrName})
}
self.submit(op, identifier=identifier)
@staticmethod
def _getDecryptedData(encData, key):
data = bytes(bytearray.fromhex(encData))
rawKey = bytes(bytearray.fromhex(key))
box = libnacl.secret.SecretBox(rawKey)
decData = box.decrypt(data).decode()
return json.loads(decData)
def hasNym(self, nym):
for txn in self.txnLog.getTxnsByType(NYM):
if txn.get(TXN_TYPE) == NYM:
return True
return False
def _statusChanged(self, old, new):
super()._statusChanged(old, new)
def start(self, loop):
super().start(loop)
if self.hasAnonCreds and self.status not in Status.going():
self.peerStack.start()
async def prod(self, limit) -> int:
s = await super().prod(limit)
if self.hasAnonCreds:
return s + await self.peerStack.service(limit)
else:
return s
def registerObserver(self, observer: Callable, name=None):
if not name:
name = uuid.uuid4()
if name in self._observers or observer in self._observerSet:
raise RuntimeError("Observer {} already registered".format(name))
self._observers[name] = observer
self._observerSet.add(observer)
def deregisterObserver(self, name):
if name not in self._observers:
raise RuntimeError("Observer {} not registered".format(name))
self._observerSet.remove(self._observers[name])
del self._observers[name]
def hasObserver(self, name):
return name in self._observerSet
class Replica(MessageProcessor):
def __init__(self,
node: 'plenum.server.node.Node',
instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router((PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit))
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# Requests that are stored by non primary replica for which it is
# expecting corresponding pre prepare requests Dictionary that stores
# a tuple of client id and request id(sequence no) as key and digest as
# value. Not creating a set of Tuple3(identifier, reqId, digest) as such a
# big hashable element is not good. Also this way we can look for the
# request on the basis of (identifier, reqId) and compare the digest with
# the received PrePrepare request's digest.
self.reqsPendingPrePrepare = {}
# type: Dict[Tuple[str, int], str]
# PREPARE that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a Request Digest
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[ReqDigest, float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a Request Digest
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[ReqDigest, float]]
self.prePrepareSeqNo = 0 # type: int
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> (digest, {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[str, Set[str]]]
self.commits = Commits()
# Set of tuples to keep track of ordered requests
self.ordered = set() # type: Set[Tuple[int, int]]
# Requests with sufficient commits so they can be ordered but have not
# received request digest from node and neither PRE-PREPARE or PREPARE.
# Key can be a digest to a tuple of viewNo and ppSeqNo
self.commitsPendedForOrdering = {}
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
def shouldParticipate(self, viewNo: int, ppSeqNo: int):
# Replica should only participating in the consensus process and the
# replica did not stash any of this request's 3-phase request
return self.node.isParticipating and (viewNo, ppSeqNo) \
not in self.stashingWhileCatchingUp
@staticmethod
def generateName(nodeName: str, instId: int):
"""
Create and return the name for a replica using its nodeName and
instanceId.
Ex: Alpha:1
"""
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".format(
self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".format(
self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicion exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicion exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo == self.viewNo
def isMsgForPrevView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
if self.isMsgForLaterView(msg):
self.discard(
msg, "Cannot get primary status for a request for a later "
"view {}. Request is {}".format(self.viewNo, msg),
logger.error)
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
logger.error("{} cannot get primary for a request for a later "
"view. Request is {}".format(self, msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
logger.debug(
"{} stashing request digest {} since it does not know "
"its primary status".format(self, (rd.identifier, rd.reqId)))
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
return self.inBoxRouter.handleAllSync(self.inBox, limit)
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}".format(
self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param msg: the ThreePhaseMsg to dispatch
:param sender: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
logger.debug("Non primary replica {} pended request for Pre "
"Prepare {}".format(self, (rd.identifier, rd.reqId)))
self.reqsPendingPrePrepare[(rd.identifier, rd.reqId)] = rd.digest
else:
self.doPrePrepare(rd)
if rd.digest in self.commitsPendedForOrdering:
viewNo, ppSeqNo, ppTime = self.commitsPendedForOrdering[rd.digest]
self.doOrder(viewNo, ppSeqNo, rd.identifier, rd.reqId, rd.digest,
ppTime)
self.commitsPendedForOrdering.pop((viewNo, ppSeqNo), None)
self.commitsPendedForOrdering.pop(rd.digest, None)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug(
"{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".format(
self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
key = (pp.viewNo, pp.ppSeqNo)
logger.debug("{} Receiving PRE-PREPARE {} at {} from {}".format(
self, key, time.perf_counter(), sender))
if self.canProcessPrePrepare(pp, sender):
if not self.node.isParticipating:
self.stashingWhileCatchingUp.add(key)
self.addToPrePrepares(pp)
if key in self.commitsPendedForOrdering or pp.digest in \
self.commitsPendedForOrdering:
self.doOrder(*key, pp.identifier, pp.reqId, pp.digest,
pp.ppTime)
self.commitsPendedForOrdering.pop(key, None)
self.commitsPendedForOrdering.pop(pp.digest, None)
# elif pp.digest in self.commitsPendedForOrdering:
# self.doOrder(*key, pp.identifier, pp.reqId, pp.digest)
# self.commitsPendedForOrdering.pop(pp.digest)
# self.commitsPendedForOrdering.pop(key, None)
logger.info("{} processed incoming PRE-PREPARE {}".format(
self, key))
def tryPrepare(self, pp: PrePrepare):
"""
Try to send the Prepare message if the PrePrepare message is ready to
be passed into the Prepare phase.
"""
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
logger.debug("{} received PREPARE {} from {}".format(
self, prepare, sender))
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
logger.debug("{} processed incoming PREPARE {}".format(
self, (prepare.viewNo, prepare.ppSeqNo)))
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning(
"{} cannot process incoming PREPARE".format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received COMMIT {} from {}".format(
self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
logger.debug("{} processed incoming COMMIT {}".format(
self, (commit.viewNo, commit.ppSeqNo)))
def tryCommit(self, prepare: Prepare):
"""
Try to commit if the Prepare message is ready to be passed into the
commit phase.
"""
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
"""
Try to order if the Commit message is ready to be ordered.
"""
canOrder, reason = self.canOrder(commit)
if canOrder:
logger.debug("{} returning request to node".format(self))
self.tryOrdering(commit)
else:
logger.trace("{} cannot return request to node: {}".format(
self, reason))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements identifier, reqId, and digest
"""
if not self.node.isParticipating:
logger.error("Non participating node is attempting PRE-PREPARE. "
"This should not happen.")
return
self.prePrepareSeqNo += 1
tm = time.time() * 1000
logger.debug("{} Sending PRE-PREPARE {} at {}".format(
self, (self.viewNo, self.prePrepareSeqNo), time.perf_counter()))
prePrepareReq = PrePrepare(self.instId, self.viewNo,
self.prePrepareSeqNo, *reqDigest, tm)
self.sentPrePrepares[self.viewNo,
self.prePrepareSeqNo] = (reqDigest, tm)
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE {} at {}".format(
self, (pp.viewNo, pp.ppSeqNo), time.perf_counter()))
prepare = Prepare(self.instId, pp.viewNo, pp.ppSeqNo, pp.digest,
pp.ppTime)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
"""
Create a commit message from the given Prepare message and trigger the
commit phase
:param p: the prepare message
"""
logger.debug("{} Sending COMMIT {} at {}".format(
self, (p.viewNo, p.ppSeqNo), time.perf_counter()))
commit = Commit(self.instId, p.viewNo, p.ppSeqNo, p.digest, p.ppTime)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool:
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# TODO: Check whether it is rejecting PRE-PREPARE from previous view
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
# A PRE-PREPARE is sent that has already been received
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
key = (pp.identifier, pp.reqId)
# A PRE-PREPARE is sent that does not match request digest
if (key in self.reqsPendingPrePrepare
and self.reqsPendingPrePrepare[key] != pp.digest):
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
self.prePrepares[(pp.viewNo, pp.ppSeqNo)] = \
((pp.identifier, pp.reqId, pp.digest), pp.ppTime)
self.dequeuePrepares(pp.viewNo, pp.ppSeqNo)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def hasPrepared(self, request) -> bool:
return self.prepares.hasPrepareFrom(request, self.name)
def canSendPrepare(self, request) -> bool:
"""
Return whether the request identified by (identifier, requestId) can
proceed to the Prepare step.
:param request: any object with identifier and requestId attributes
"""
return self.shouldParticipate(request.viewNo, request.ppSeqNo) and \
self.node.requests.canPrepare(request, self.f + 1) and \
not self.hasPrepared(request)
def isValidPrepare(self, prepare: Prepare, sender: str) -> bool:
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT,
prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
return False
elif prepare.digest != ppReqs[key][0][2]:
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG,
prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT,
prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT,
prepare)
elif prepare.digest != ppReqs[key][0][2]:
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG,
prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def hasCommitted(self, request) -> bool:
return self.commits.hasCommitFrom(
ThreePhaseKey(request.viewNo, request.ppSeqNo), self.name)
def canCommit(self, prepare: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param prepare: the PREPARE
"""
return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \
self.prepares.hasQuorum(prepare, self.f) and \
not self.hasCommitted(prepare)
def isValidCommit(self, commit: Commit, sender: str) -> bool:
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
primaryStatus = self.isPrimaryForMsg(commit)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
key = (commit.viewNo, commit.ppSeqNo)
if (key not in self.prepares
and key not in self.preparesWaitingForPrePrepare):
logger.debug(
"{} rejecting commit {} due to lack of prepares".format(
self, key))
# raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
return False
elif self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
elif commit.digest != self.getDigestFromPrepare(*key):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
elif key in ppReqs and commit.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG, commit)
else:
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def hasOrdered(self, request) -> bool:
return (request.viewNo, request.ppSeqNo) in self.ordered
def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
if not self.commits.hasQuorum(commit, self.f):
return False, "no quorum: {} commits where f is {}".\
format(commit, self.f)
if self.hasOrdered(commit):
return False, "already ordered"
return True, None
def tryOrdering(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
logger.debug("{} ordering commit {}".format(self, key))
primaryStatus = self.isPrimaryForMsg(commit)
if primaryStatus is True:
(identifier, reqId, digest), tm = self.sentPrePrepares[key]
elif primaryStatus is False:
# When the node received PREPARE requests and PRE-PREPARE request
if key in self.prePrepares:
(identifier, reqId, digest), tm = self.prePrepares[key]
else:
digest = self.getDigestFromPrepare(*key)
if digest:
for (cid, rid), dgst in self.reqsPendingPrePrepare.items():
if digest == dgst:
identifier, reqId = cid, rid
break
else:
logger.info("{} cannot find digest {} in "
"reqsPendingPrePrepare for viewNo {} and "
"ppSeqNo {}".format(self, digest, *key))
self.commitsPendedForOrdering[digest] = (*key,
commit.ppTime)
self.commitsPendedForOrdering[key] = commit.ppTime
return
else:
logger.info("{} cannot find digest {} in "
"received PREPAREs for viewNo {} and "
"ppSeqNo {}".format(self, digest, *key))
self.commitsPendedForOrdering[key] = commit.ppTime
return
else:
self.discard(
commit, "{}'s primary status found None while returning "
"request {} to node".format(self, commit), logger.warning)
return
self.doOrder(*key, identifier, reqId, digest, commit.ppTime)
def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime):
key = (viewNo, ppSeqNo)
self.addToOrdered(*key)
ordered = Ordered(self.instId, viewNo, identifier, reqId, digest,
ppTime)
self.send(ordered, TPCStat.OrderSent)
if key in self.stashingWhileCatchingUp:
self.stashingWhileCatchingUp.remove(key)
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrepare(self, request: Prepare, sender: str):
logger.debug("Queueing prepares due to unavailability of "
"pre-prepare. request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
self.processPrepare(prepare, sender)
i += 1
logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for"
" view no {} and seq no {}".format(
self, i, viewNo, ppSeqNo))
def getDigestFromPrepare(self, viewNo: int, ppSeqNo: int) -> Optional[str]:
key = (viewNo, ppSeqNo)
if key in self.prepares:
return self.prepares[key].digest
elif key in self.preparesWaitingForPrePrepare:
prepare, _ = self.preparesWaitingForPrePrepare[key][0]
return prepare.digest
else:
logger.debug("{} could not find digest for PRE-PREPARE {}".format(
self, key))
return None
def send(self, msg, stat) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.display("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
self.stats.inc(stat)
self.outBox.append(msg)
class Replica(MessageProcessor):
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Do not know which node is primary or even if any node is primary yet
# self._isPrimary = None # type: Optional[bool]
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# Requests that are stored by non primary replica for which it is
# expecting corresponding pre prepare requests Dictionary that stores
# a tuple of client id and request id(sequence no) as key and digest as
# value. Not creating a set of Tuple3(clientId, reqId, digest) as such a
# big hashable element is not good. Also this way we can look for the
# request on the basis of (clientId, reqId) and compare the digest with
# the received PrePrepare request's digest.
self.reqsPendingPrePrepare = {}
# type: Dict[Tuple[str, int], str]
# PREPARE that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# Requests that are stored by primary replica which it has broadcasted
# to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a Request Digest
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], ReqDigest]
# Dictionary of received PrePrepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a Request Digest
self.prePrepares = {}
# type: Dict[Tuple[int, int], ReqDigest]
self.prePrepareSeqNo = 0 # type: int
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> (digest, {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[str, Set[str]]]
self.commits = Commits()
# Set of tuples to keep track of ordered requests
self.ordered = set() # type: Set[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
@staticmethod
def generateName(nodeName: str, instId: int):
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".
format(self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".
format(self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicous exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicous exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
# Assumes request has an attribute view no
return msg.viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
# Assumes request has an attribute view no
return msg.viewNo == self.viewNo
def isMsgForPastView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
# Assumes request has an attribute view no
return msg.viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
# Assumes request has an attribute view no
if self.isMsgForLaterView(msg):
RuntimeError("Cannot get primary status for a request for a later "
"view. Request is {}".format(msg))
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
RuntimeError("Cannot get primary for a request for a later "
"view. Request is {}".format(msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
return self.inBoxRouter.handleAllSync(self.inBox, limit)
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}"
.format(self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
# Sacrificing brevity for efficiency.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param request: the ThreePhaseRequest to dispatch
:param senderRep: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
logger.debug("Non primary replica {} pended request for Pre "
"Prepare {}".format(self, (rd.clientId, rd.reqId)))
self.reqsPendingPrePrepare[(rd.clientId, rd.reqId)] = rd.digest
else:
self.doPrePrepare(rd)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug("{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".
format(self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
logger.debug("{} Receiving PRE-PREPARE at {}".
format(self, time.perf_counter()))
if self.canProcessPrePrepare(pp, sender):
self.addToPrePrepares(pp)
def tryPrepare(self, pp: PrePrepare):
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning("{} cannot process incoming PREPARE".format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received commit {} from {}".format(self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
def tryCommit(self, prepare: Prepare):
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
if self.canOrder(commit):
logging.debug("{} returning request to node".format(self))
self.doOrder(commit)
else:
logger.trace("{} cannot return request to node".format(self))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements clientId, reqId, and digest
"""
logger.debug("{} Sending PRE-PREPARE at {}".
format(self, time.perf_counter()))
self.prePrepareSeqNo += 1
prePrepareReq = PrePrepare(self.instId,
self.viewNo,
self.prePrepareSeqNo,
*reqDigest)
self.sentPrePrepares[self.viewNo, self.prePrepareSeqNo] = reqDigest
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE at {}".
format(self, time.perf_counter()))
prepare = Prepare(self.instId,
pp.viewNo,
pp.ppSeqNo,
pp.digest)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
commit = Commit(self.instId,
p.viewNo,
p.ppSeqNo,
p.digest)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str):
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
#TODO: Fix this, how to check last preprepare seq num
# if self.prePrepares:
# lastProcessedPrePrepareSeqNo = max([key[1] for key in self.prePrepares.keys()])
# if pp.ppSeqNo > lastProcessedPrePrepareSeqNo + 1:
# raise SuspiciousNode(sender, Suspicions.WRONG_PPSEQ_NO, pp)
key = (pp.clientId, pp.reqId)
if (key in self.reqsPendingPrePrepare and
self.reqsPendingPrePrepare[key] != pp.digest):
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
self.prePrepares[(pp.viewNo, pp.ppSeqNo)] = \
(pp.clientId, pp.reqId, pp.digest)
self.dequeuePrepares(pp.viewNo, pp.ppSeqNo)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def canSendPrepare(self, request) -> None:
"""
Return whether the request identified by (clientId, requestId) can
proceed to the Prepare step.
:param request: any object with clientId and requestId attributes
"""
return self.node.requests.canPrepare(request, self.f + 1)
def isValidPrepare(self, prepare: Prepare, sender: str):
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
elif prepare.digest != ppReqs[key][2]:
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT, prepare)
elif prepare.digest != ppReqs[key][2]:
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def canCommit(self, request: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param request: the PREPARE
"""
return self.prepares.hasQuorum(request, self.f) and \
not self.commits.hasCommitFrom(ThreePhaseKey(
request.viewNo, request.ppSeqNo),
self.name)
def isValidCommit(self, commit: Commit, sender: str):
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
key = (commit.viewNo, commit.ppSeqNo)
if (key not in self.prepares and
key not in self.preparesWaitingForPrePrepare):
raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
if self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
if commit.digest != self.getDigestFromPrepare(*key):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def canOrder(self, commit: Commit) -> bool:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
return self.commits.hasQuorum(commit, self.f) and \
(commit.viewNo, commit.ppSeqNo) not in self.ordered
def doOrder(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(commit)
if primaryStatus is True:
clientId, reqId, digest = self.sentPrePrepares[key]
elif primaryStatus is False:
# When the node received PREPARE requests and PRE-PREPARE request
if key in self.prePrepares:
clientId, reqId, digest = self.prePrepares[key]
else:
digest = self.getDigestFromPrepare(*key)
for (cid, rid), dgst \
in self.reqsPendingPrePrepare.items():
if digest == dgst:
clientId, reqId = cid, rid
break
else:
self.discard(commit,
"{}'s primary status found None while returning "
"request {} to node".format(self, commit),
logger.warning)
self.addToOrdered(commit.viewNo, commit.ppSeqNo)
ordered = Ordered(self.instId,
commit.viewNo,
clientId,
reqId,
digest)
self.send(ordered, TPCStat.OrderSent)
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrepare(self, request: Prepare, sender: str):
logging.debug("Queueing prepares due to unavailability of "
"pre-prepare. request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
logging.debug("Processing prepares waiting for pre-prepare for "
"view no {} and seq no {}".format(viewNo, ppSeqNo))
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
self.processPrepare(prepare, sender)
def getDigestFromPrepare(self, viewNo: int, ppSeqNo: int) -> Optional[str]:
key = (viewNo, ppSeqNo)
if key in self.prepares:
return self.prepares[key].digest
elif key in self.preparesWaitingForPrePrepare:
prepare, _ = self.preparesWaitingForPrePrepare[key][0]
return prepare.digest
else:
return None
def send(self, msg, stat) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.debug("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
self.stats.inc(stat)
self.outBox.append(msg)
def __init__(self,
node: 'plenum.server.node.Node',
instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router((PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit))
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# Requests that are stored by non primary replica for which it is
# expecting corresponding pre prepare requests Dictionary that stores
# a tuple of client id and request id(sequence no) as key and digest as
# value. Not creating a set of Tuple3(identifier, reqId, digest) as such a
# big hashable element is not good. Also this way we can look for the
# request on the basis of (identifier, reqId) and compare the digest with
# the received PrePrepare request's digest.
self.reqsPendingPrePrepare = {}
# type: Dict[Tuple[str, int], str]
# PREPARE that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a Request Digest
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[ReqDigest, float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a Request Digest
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[ReqDigest, float]]
self.prePrepareSeqNo = 0 # type: int
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> (digest, {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[str, Set[str]]]
self.commits = Commits()
# Set of tuples to keep track of ordered requests
self.ordered = set() # type: Set[Tuple[int, int]]
# Requests with sufficient commits so they can be ordered but have not
# received request digest from node and neither PRE-PREPARE or PREPARE.
# Key can be a digest to a tuple of viewNo and ppSeqNo
self.commitsPendedForOrdering = {}
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Do not know which node is primary or even if any node is primary yet
# self._isPrimary = None # type: Optional[bool]
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# Requests that are stored by non primary replica for which it is
# expecting corresponding pre prepare requests Dictionary that stores
# a tuple of client id and request id(sequence no) as key and digest as
# value. Not creating a set of Tuple3(clientId, reqId, digest) as such a
# big hashable element is not good. Also this way we can look for the
# request on the basis of (clientId, reqId) and compare the digest with
# the received PrePrepare request's digest.
self.reqsPendingPrePrepare = {}
# type: Dict[Tuple[str, int], str]
# PREPARE that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# Requests that are stored by primary replica which it has broadcasted
# to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a Request Digest
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], ReqDigest]
# Dictionary of received PrePrepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a Request Digest
self.prePrepares = {}
# type: Dict[Tuple[int, int], ReqDigest]
self.prePrepareSeqNo = 0 # type: int
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> (digest, {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[str, Set[str]]]
self.commits = Commits()
# Set of tuples to keep track of ordered requests
self.ordered = set() # type: Set[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
class Client(PlenumClient):
anoncredsAreSetUp = False
def __init__(self,
name: str=None,
nodeReg: Dict[str, HA]=None,
ha: Union[HA, Tuple[str, int]]=None,
peerHA: Union[HA, Tuple[str, int]]=None,
basedirpath: str=None,
config=None,
sighex: str=None):
self.config = config or getConfig()
self.setupAnoncreds()
basedirpath = basedirpath or os.path.join(self.config.CLI_NETWORK_DIR, self.config.NETWORK_NAME)
super().__init__(name,
nodeReg,
ha,
basedirpath,
config=config,
sighex=sighex)
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auth_mode=AuthMode.ALLOW_ANY.value)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = self.peerStackClass(
stackargs, msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
# To let client send this transactions to just one node
self._read_only_requests = {GET_NYM,
GET_ATTR,
GET_CLAIM_DEF,
GET_SCHEMA}
@property
def peerStackClass(self):
return SimpleZStack
def setupAnoncreds(self):
if self.anoncredsAreSetUp is False:
writeAnonCredPlugin(os.path.expanduser(self.config.CLI_BASE_DIR))
# This is to setup anoncreds wallet related custom jsonpickle handlers to
# serialize/deserialize it properly
setUpJsonpickle()
WALLET_RAW_MIGRATORS.append(migrate_indy_wallet_raw)
self.anoncredsAreSetUp = True
def handlePeerMessage(self, msg):
"""
Use the peerMsgRouter to pass the messages to the correct
function that handles them
:param msg: the P2P client message.
"""
return self.peerMsgRouter.handle(msg)
def getReqRepStore(self):
return ClientReqRepStoreFile(self.ledger_dir)
def getTxnLogStore(self):
return ClientTxnLog(self.ledger_dir)
def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None:
msg, sender = wrappedMsg
# excludeGetTxns = (msg.get(OP_FIELD_NAME) == REPLY and
# msg[f.RESULT.nm].get(TXN_TYPE) == GET_TXNS)
excludeReqAcks = msg.get(OP_FIELD_NAME) == REQACK
excludeReqNacks = msg.get(OP_FIELD_NAME) == REQNACK
excludeReply = msg.get(OP_FIELD_NAME) == REPLY
excludeReject = msg.get(OP_FIELD_NAME) == REJECT
excludeFromCli = excludeFromCli or excludeReqAcks or excludeReqNacks \
or excludeReply or excludeReject
super().handleOneNodeMsg(wrappedMsg, excludeFromCli)
if OP_FIELD_NAME not in msg:
logger.error("Op absent in message {}".format(msg))
def requestConfirmed(self, key) -> bool:
return self.txnLog.hasTxnWithReqId(key)
def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
return super().hasConsensus(identifier, reqId)
def prepare_for_state(self, result):
request_type = result[TYPE]
if request_type == GET_NYM:
return domain.prepare_get_nym_for_state(result)
if request_type == GET_ATTR:
attr_type, path, value, hashed_value, value_bytes = \
domain.prepare_get_attr_for_state(result)
return path, value_bytes
if request_type == GET_CLAIM_DEF:
return domain.prepare_get_claim_def_for_state(result)
if request_type == GET_SCHEMA:
return domain.prepare_get_schema_for_state(result)
raise ValueError("Cannot make state key for "
"request of type {}"
.format(request_type))
def getTxnsByType(self, txnType):
return self.txnLog.getTxnsByType(txnType)
# TODO: Just for now. Remove it later
def doAttrDisclose(self, origin, target, txnId, key):
box = libnacl.public.Box(b58decode(origin), b58decode(target))
data = json.dumps({TXN_ID: txnId, SKEY: key})
nonce, boxedMsg = box.encrypt(data.encode(), pack_nonce=False)
op = {
TARGET_NYM: target,
TXN_TYPE: DISCLO,
NONCE: b58encode(nonce).decode("utf-8"),
DATA: b58encode(boxedMsg).decode("utf-8")
}
self.submit(op, identifier=origin)
def doGetAttributeTxn(self, identifier, attrName):
op = {
TARGET_NYM: identifier,
TXN_TYPE: GET_ATTR,
DATA: json.dumps({"name": attrName})
}
self.submit(op, identifier=identifier)
@staticmethod
def _getDecryptedData(encData, key):
data = bytes(bytearray.fromhex(encData))
rawKey = bytes(bytearray.fromhex(key))
box = libnacl.secret.SecretBox(rawKey)
decData = box.decrypt(data).decode()
return json.loads(decData)
def hasNym(self, nym):
for txn in self.txnLog.getTxnsByType(NYM):
if get_type(txn) == NYM:
return True
return False
def _statusChanged(self, old, new):
super()._statusChanged(old, new)
def start(self, loop):
super().start(loop)
if self.hasAnonCreds and self.status not in Status.going():
self.peerStack.start()
async def prod(self, limit) -> int:
s = await super().prod(limit)
if self.hasAnonCreds:
return s + await self.peerStack.service(limit)
else:
return s
def __init__(self,
node: 'plenum.server.node.Node',
instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
HasActionQueue.__init__(self)
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router((PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit))
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {
} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
class Replica(HasActionQueue, MessageProcessor):
def __init__(self,
node: 'plenum.server.node.Node',
instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
HasActionQueue.__init__(self)
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router((PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit))
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {
} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
@property
def h(self) -> int:
return self._h
@h.setter
def h(self, n):
self._h = n
self.H = self._h + self.config.LOG_SIZE
@property
def requests(self):
return self.node.requests
def shouldParticipate(self, viewNo: int, ppSeqNo: int):
# Replica should only participating in the consensus process and the
# replica did not stash any of this request's 3-phase request
return self.node.isParticipating and (viewNo, ppSeqNo) \
not in self.stashingWhileCatchingUp
@staticmethod
def generateName(nodeName: str, instId: int):
"""
Create and return the name for a replica using its nodeName and
instanceId.
Ex: Alpha:1
"""
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".format(
self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".format(
self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicion exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicion exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo == self.viewNo
def isMsgForPrevView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
if self.isMsgForLaterView(msg):
self.discard(
msg, "Cannot get primary status for a request for a later "
"view {}. Request is {}".format(self.viewNo, msg),
logger.error)
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
logger.error("{} cannot get primary for a request for a later "
"view. Request is {}".format(self, msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
logger.debug(
"{} stashing request digest {} since it does not know "
"its primary status".format(self, (rd.identifier, rd.reqId)))
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
r = self.inBoxRouter.handleAllSync(self.inBox, limit)
r += self._serviceActions()
return r
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}".format(
self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param msg: the ThreePhaseMsg to dispatch
:param sender: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
if self.isPpSeqNoAcceptable(msg.ppSeqNo):
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
else:
logger.debug("{} stashing 3 phase message {} since ppSeqNo {} is "
"not between {} and {}".format(
self, msg, msg.ppSeqNo, self.h, self.H))
self.stashingWhileOutsideWaterMarks.append((msg, sender))
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
self.dequeuePrePrepare(rd.identifier, rd.reqId)
else:
self.doPrePrepare(rd)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug(
"{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".format(
self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
key = (pp.viewNo, pp.ppSeqNo)
logger.debug("{} Receiving PRE-PREPARE{} at {} from {}".format(
self, key, time.perf_counter(), sender))
if self.canProcessPrePrepare(pp, sender):
if not self.node.isParticipating:
self.stashingWhileCatchingUp.add(key)
self.addToPrePrepares(pp)
logger.info("{} processed incoming PRE-PREPARE{}".format(
self, key))
def tryPrepare(self, pp: PrePrepare):
"""
Try to send the Prepare message if the PrePrepare message is ready to
be passed into the Prepare phase.
"""
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
logger.debug("{} received PREPARE{} from {}".format(
self, (prepare.viewNo, prepare.ppSeqNo), sender))
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
logger.debug("{} processed incoming PREPARE {}".format(
self, (prepare.viewNo, prepare.ppSeqNo)))
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning(
"{} cannot process incoming PREPARE".format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received COMMIT {} from {}".format(
self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
logger.debug("{} processed incoming COMMIT{}".format(
self, (commit.viewNo, commit.ppSeqNo)))
def tryCommit(self, prepare: Prepare):
"""
Try to commit if the Prepare message is ready to be passed into the
commit phase.
"""
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
"""
Try to order if the Commit message is ready to be ordered.
"""
canOrder, reason = self.canOrder(commit)
if canOrder:
logger.debug("{} returning request to node".format(self))
self.tryOrdering(commit)
else:
logger.trace("{} cannot return request to node: {}".format(
self, reason))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements identifier, reqId, and digest
"""
if not self.node.isParticipating:
logger.error("Non participating node is attempting PRE-PREPARE. "
"This should not happen.")
return
if self.lastPrePrepareSeqNo == self.H:
logger.debug("{} stashing PRE-PREPARE {} since outside greater "
"than high water mark {}".format(
self, (self.viewNo, self.lastPrePrepareSeqNo + 1),
self.H))
self.stashingWhileOutsideWaterMarks.append(reqDigest)
return
self.lastPrePrepareSeqNo += 1
tm = time.time() * 1000
logger.debug("{} Sending PRE-PREPARE {} at {}".format(
self, (self.viewNo, self.lastPrePrepareSeqNo),
time.perf_counter()))
prePrepareReq = PrePrepare(self.instId, self.viewNo,
self.lastPrePrepareSeqNo, *reqDigest, tm)
self.sentPrePrepares[self.viewNo,
self.lastPrePrepareSeqNo] = (reqDigest.key, tm)
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE {} at {}".format(
self, (pp.viewNo, pp.ppSeqNo), time.perf_counter()))
prepare = Prepare(self.instId, pp.viewNo, pp.ppSeqNo, pp.digest,
pp.ppTime)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
"""
Create a commit message from the given Prepare message and trigger the
commit phase
:param p: the prepare message
"""
logger.debug("{} Sending COMMIT{} at {}".format(
self, (p.viewNo, p.ppSeqNo), time.perf_counter()))
commit = Commit(self.instId, p.viewNo, p.ppSeqNo, p.digest, p.ppTime)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool:
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# TODO: Check whether it is rejecting PRE-PREPARE from previous view
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
# A PRE-PREPARE is sent that has already been received
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
key = (pp.identifier, pp.reqId)
if not self.requests.isFinalised(key):
self.enqueuePrePrepare(pp, sender)
return False
# A PRE-PREPARE is sent that does not match request digest
if self.requests.digest(key) != pp.digest:
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
key = (pp.viewNo, pp.ppSeqNo)
self.prePrepares[key] = \
((pp.identifier, pp.reqId), pp.ppTime)
self.dequeuePrepares(*key)
self.dequeueCommits(*key)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def hasPrepared(self, request) -> bool:
return self.prepares.hasPrepareFrom(request, self.name)
def canSendPrepare(self, request) -> bool:
"""
Return whether the request identified by (identifier, requestId) can
proceed to the Prepare step.
:param request: any object with identifier and requestId attributes
"""
return self.shouldParticipate(request.viewNo, request.ppSeqNo) \
and not self.hasPrepared(request) \
and self.requests.isFinalised((request.identifier,
request.reqId))
def isValidPrepare(self, prepare: Prepare, sender: str) -> bool:
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT,
prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
return False
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG,
prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT,
prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT,
prepare)
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG,
prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def hasCommitted(self, request) -> bool:
return self.commits.hasCommitFrom(
ThreePhaseKey(request.viewNo, request.ppSeqNo), self.name)
def canCommit(self, prepare: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param prepare: the PREPARE
"""
return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \
self.prepares.hasQuorum(prepare, self.f) and \
not self.hasCommitted(prepare)
def isValidCommit(self, commit: Commit, sender: str) -> bool:
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
primaryStatus = self.isPrimaryForMsg(commit)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
key = (commit.viewNo, commit.ppSeqNo)
if key not in ppReqs:
self.enqueueCommit(commit, sender)
return False
if (key not in self.prepares
and key not in self.preparesWaitingForPrePrepare):
logger.debug(
"{} rejecting COMMIT{} due to lack of prepares".format(
self, key))
# raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
return False
elif self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
elif commit.digest != self.getDigestFor3PhaseKey(ThreePhaseKey(*key)):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
elif key in ppReqs and commit.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG, commit)
else:
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def hasOrdered(self, viewNo, ppSeqNo) -> bool:
return (viewNo, ppSeqNo) in self.ordered
def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
if not self.commits.hasQuorum(commit, self.f):
return False, "no quorum: {} commits where f is {}".\
format(commit, self.f)
if self.hasOrdered(commit.viewNo, commit.ppSeqNo):
return False, "already ordered"
if not self.isNextInOrdering(commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if viewNo not in self.stashedCommitsForOrdering:
self.stashedCommitsForOrdering[viewNo] = {}
self.stashedCommitsForOrdering[viewNo][ppSeqNo] = commit
# self._schedule(self.orderStashedCommits, 2)
self.startRepeating(self.orderStashedCommits, 2)
return False, "stashing {} since out of order".\
format(commit)
return True, None
def isNextInOrdering(self, commit: Commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if self.ordered and self.ordered[-1] == (viewNo, ppSeqNo - 1):
return True
for (v, p) in self.commits:
if v < viewNo:
# Have commits from previous view that are unordered.
# TODO: Question: would commits be always ordered, what if
# some are never ordered and its fine, go to PBFT.
return False
if v == viewNo and p < ppSeqNo and (v, p) not in self.ordered:
# If unordered commits are found with lower ppSeqNo then this
# cannot be ordered.
return False
# TODO: Revisit PBFT paper, how to make sure that last request of the
# last view has been ordered? Need change in `VIEW CHANGE` mechanism.
# Somehow view change needs to communicate what the last request was.
# Also what if some COMMITs were completely missed in the same view
return True
def orderStashedCommits(self):
# TODO: What if the first few commits were out of order and stashed?
# `self.ordered` would be empty
if self.ordered:
lastOrdered = self.ordered[-1]
vToRemove = set()
for v in self.stashedCommitsForOrdering:
if v < lastOrdered[0] and self.stashedCommitsForOrdering[v]:
raise RuntimeError(
"{} found commits from previous view {}"
" that were not ordered but last ordered"
" is {}".format(self, v, lastOrdered))
pToRemove = set()
for p, commit in self.stashedCommitsForOrdering[v].items():
if (v == lastOrdered[0] and lastOrdered == (v, p - 1)) or \
(v > lastOrdered[0] and
self.isLowestCommitInView(commit)):
logger.debug("{} ordering stashed commit {}".format(
self, commit))
if self.tryOrdering(commit):
lastOrdered = (v, p)
pToRemove.add(p)
for p in pToRemove:
del self.stashedCommitsForOrdering[v][p]
if not self.stashedCommitsForOrdering[v]:
vToRemove.add(v)
for v in vToRemove:
del self.stashedCommitsForOrdering[v]
# if self.stashedCommitsForOrdering:
# self._schedule(self.orderStashedCommits, 2)
if not self.stashedCommitsForOrdering:
self.stopRepeating(self.orderStashedCommits)
def isLowestCommitInView(self, commit):
# TODO: Assumption: This assumes that at least one commit that was sent
# for any request by any node has been received in the view of this
# commit
ppSeqNos = []
for v, p in self.commits:
if v == commit.viewNo:
ppSeqNos.append(p)
return min(ppSeqNos) == commit.ppSeqNo if ppSeqNos else True
def tryOrdering(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
logger.debug("{} trying to order COMMIT{}".format(self, key))
reqKey = self.getReqKeyFrom3PhaseKey(key) # type: Tuple
digest = self.getDigestFor3PhaseKey(key)
if not digest:
logger.error(
"{} did not find digest for {}, request key {}".format(
self, key, reqKey))
return
self.doOrder(*key, *reqKey, digest, commit.ppTime)
return True
def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime):
key = (viewNo, ppSeqNo)
self.addToOrdered(*key)
ordered = Ordered(self.instId, viewNo, identifier, reqId, ppTime)
# TODO: Should not order or add to checkpoint while syncing
# 3 phase state.
self.send(ordered, TPCStat.OrderSent)
if key in self.stashingWhileCatchingUp:
self.stashingWhileCatchingUp.remove(key)
logger.debug("{} ordered request {}".format(self, (viewNo, ppSeqNo)))
self.addToCheckpoint(ppSeqNo, digest)
def processCheckpoint(self, msg: Checkpoint, sender: str):
if self.checkpoints:
seqNo = msg.seqNo
_, firstChk = self.firstCheckPoint
if firstChk.isStable:
if firstChk.seqNo == seqNo:
self.discard(msg,
reason="Checkpoint already stable",
logMethod=logger.debug)
return
if firstChk.seqNo > seqNo:
self.discard(msg,
reason="Higher stable checkpoint present",
logMethod=logger.debug)
return
for state in self.checkpoints.values():
if state.seqNo == seqNo:
if state.digest == msg.digest:
state.receivedDigests[sender] = msg.digest
break
else:
logger.error("{} received an incorrect digest {} for "
"checkpoint {} from {}".format(
self, msg.digest, seqNo, sender))
return
if len(state.receivedDigests) == 2 * self.f:
self.markCheckPointStable(msg.seqNo)
else:
self.discard(msg,
reason="No checkpoints present to tally",
logMethod=logger.warn)
def _newCheckpointState(self, ppSeqNo, digest) -> CheckpointState:
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ - 1
logger.debug("{} adding new checkpoint state for {}".format(
self, (s, e)))
state = CheckpointState(ppSeqNo, [
digest,
], None, {}, False)
self.checkpoints[s, e] = state
return state
def addToCheckpoint(self, ppSeqNo, digest):
for (s, e) in self.checkpoints.keys():
if s <= ppSeqNo <= e:
state = self.checkpoints[s, e] # type: CheckpointState
state.digests.append(digest)
state = updateNamedTuple(state, seqNo=ppSeqNo)
self.checkpoints[s, e] = state
break
else:
state = self._newCheckpointState(ppSeqNo, digest)
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ
if len(state.digests) == self.config.CHK_FREQ:
state = updateNamedTuple(state,
digest=serialize(state.digests),
digests=[])
self.checkpoints[s, e] = state
self.send(
Checkpoint(self.instId, self.viewNo, ppSeqNo, state.digest))
def markCheckPointStable(self, seqNo):
previousCheckpoints = []
for (s, e), state in self.checkpoints.items():
if e == seqNo:
state = updateNamedTuple(state, isStable=True)
self.checkpoints[s, e] = state
break
else:
previousCheckpoints.append((s, e))
else:
logger.error("{} could not find {} in checkpoints".format(
self, seqNo))
return
self.h = seqNo
for k in previousCheckpoints:
logger.debug("{} removing previous checkpoint {}".format(self, k))
self.checkpoints.pop(k)
self.gc(seqNo)
logger.debug("{} marked stable checkpoint {}".format(self, (s, e)))
self.processStashedMsgsForNewWaterMarks()
def gc(self, tillSeqNo):
logger.debug("{} cleaning up till {}".format(self, tillSeqNo))
tpcKeys = set()
reqKeys = set()
for (v, p), (reqKey, _) in self.sentPrePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
for (v, p), (reqKey, _) in self.prePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
logger.debug("{} found {} 3 phase keys to clean".format(
self, len(tpcKeys)))
logger.debug("{} found {} request keys to clean".format(
self, len(reqKeys)))
for k in tpcKeys:
self.sentPrePrepares.pop(k, None)
self.prePrepares.pop(k, None)
self.prepares.pop(k, None)
self.commits.pop(k, None)
if k in self.ordered:
self.ordered.remove(k)
for k in reqKeys:
self.requests.pop(k, None)
def processStashedMsgsForNewWaterMarks(self):
while self.stashingWhileOutsideWaterMarks:
item = self.stashingWhileOutsideWaterMarks.pop()
logger.debug("{} processing stashed item {} after new stable "
"checkpoint".format(self, item))
if isinstance(item, ReqDigest):
self.doPrePrepare(item)
elif isinstance(item, tuple) and len(tuple) == 2:
self.dispatchThreePhaseMsg(*item)
else:
logger.error("{} cannot process {} "
"from stashingWhileOutsideWaterMarks".format(
self, item))
@property
def firstCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(0)
@property
def lastCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(-1)
def isPpSeqNoAcceptable(self, ppSeqNo: int):
return self.h < ppSeqNo <= self.H
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrePrepare(self, request: PrePrepare, sender: str):
logger.debug(
"Queueing pre-prepares due to unavailability of finalised "
"Request. Request {} from {}".format(request, sender))
key = (request.identifier, request.reqId)
if key not in self.prePreparesPendingReqDigest:
self.prePreparesPendingReqDigest[key] = []
self.prePreparesPendingReqDigest[key].append((request, sender))
def dequeuePrePrepare(self, identifier: int, reqId: int):
key = (identifier, reqId)
if key in self.prePreparesPendingReqDigest:
pps = self.prePreparesPendingReqDigest[key]
for (pp, sender) in pps:
logger.debug("{} popping stashed PRE-PREPARE{}".format(
self, key))
if pp.digest == self.requests.digest(key):
self.prePreparesPendingReqDigest.pop(key)
self.processPrePrepare(pp, sender)
logger.debug(
"{} processed {} PRE-PREPAREs waiting for finalised "
"request for identifier {} and reqId {}".format(
self, pp, identifier, reqId))
break
def enqueuePrepare(self, request: Prepare, sender: str):
logger.debug("Queueing prepares due to unavailability of PRE-PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
logger.debug("{} popping stashed PREPARE{}".format(self, key))
self.processPrepare(prepare, sender)
i += 1
self.preparesWaitingForPrePrepare.pop(key)
logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for"
" view no {} and seq no {}".format(
self, i, viewNo, ppSeqNo))
def enqueueCommit(self, request: Commit, sender: str):
logger.debug("Queueing commit due to unavailability of PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.commitsWaitingForPrepare:
self.commitsWaitingForPrepare[key] = deque()
self.commitsWaitingForPrepare[key].append((request, sender))
def dequeueCommits(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.commitsWaitingForPrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.commitsWaitingForPrepare[key]:
commit, sender = self.commitsWaitingForPrepare[key].popleft()
logger.debug("{} popping stashed COMMIT{}".format(self, key))
self.processCommit(commit, sender)
i += 1
self.commitsWaitingForPrepare.pop(key)
logger.debug("{} processed {} COMMITs waiting for PREPARE for"
" view no {} and seq no {}".format(
self, i, viewNo, ppSeqNo))
def getDigestFor3PhaseKey(self, key: ThreePhaseKey) -> Optional[str]:
reqKey = self.getReqKeyFrom3PhaseKey(key)
digest = self.requests.digest(reqKey)
if not digest:
logger.debug("{} could not find digest in sent or received "
"PRE-PREPAREs or PREPAREs for 3 phase key {} and req "
"key {}".format(self, key, reqKey))
return None
else:
return digest
def getReqKeyFrom3PhaseKey(self, key: ThreePhaseKey):
reqKey = None
if key in self.sentPrePrepares:
reqKey = self.sentPrePrepares[key][0]
elif key in self.prePrepares:
reqKey = self.prePrepares[key][0]
elif key in self.prepares:
reqKey = self.prepares[key][0]
else:
logger.debug(
"Could not find request key for 3 phase key {}".format(key))
return reqKey
@property
def threePhaseState(self):
# TODO: This method is incomplete
# Gets the current stable and unstable checkpoints and creates digest
# of unstable checkpoints
if self.checkpoints:
pass
else:
state = []
return ThreePCState(self.instId, state)
def process3PhaseState(self, msg: ThreePCState, sender: str):
# TODO: This is not complete
pass
def send(self, msg, stat=None) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.display("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
if stat:
self.stats.inc(stat)
self.outBox.append(msg)
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
