def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = [] self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector()
def __init__(self, tmpdir): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = [] self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.replicas = [ Replica(node=self, instId=0, isMaster=True), Replica(node=self, instId=1, isMaster=False), Replica(node=self, instId=2, isMaster=False), ] self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.config = getConfig() # TODO do we need fake object here? self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self)
def __init__(self, tmpdir): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = [] self.viewNo = 0 self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.replicas = [ Replica(node=self, instId=0, isMaster=True), Replica(node=self, instId=1, isMaster=False), Replica(node=self, instId=2, isMaster=False), ] self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_change_in_progress = True self.propagate_primary = False
class FakeNode: ledger_ids = [0] def __init__(self, tmpdir): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = [] self.viewNo = 0 self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.replicas = [ Replica(node=self, instId=0, isMaster=True), Replica(node=self, instId=1, isMaster=False), Replica(node=self, instId=2, isMaster=False), ] self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_change_in_progress = True self.propagate_primary = False def get_name_by_rank(self, name, nodeReg=None): # This is used only for getting name of next primary, so # it just returns a constant return 'Node2' def primary_selected(self, instance_id): self._found = True def is_primary_found(self): return self._found @property def master_primary_name(self) -> Optional[str]: nm = self.replicas[0].primaryName if nm: return Replica.getNodeName(nm) @property def master_replica(self): return self.replicas[0] @property def is_synced(self): return self.mode >= Mode.synced
def __init__(self, tmpdir, config=None): node_names = ['Node1', 'Node2', 'Node3', 'Node4'] self.basedirpath = tmpdir self.name = node_names[0] self.viewNo = 0 self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = node_names self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.poolManager = FakeSomething( node_names_ordered_by_rank=lambda: node_names) self.mode = Mode.starting self.monitor = FakeSomething(isMasterDegraded=lambda: False) self.config = config or getConfigOnce() self.nodeStatusDB = None self.quorums = Quorums(self.totalNodes) self.nodestack = FakeSomething(connecteds=set(self.allNodeNames)) self.write_manager = FakeSomething( node_reg_handler=NodeRegHandler(self.db_manager)) self.primaries_selector = RoundRobinConstantNodesPrimariesSelector( node_names) self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config) } self.requiredNumberOfInstances = 2 self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.metrics = NullMetricsCollector() # For catchup testing self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.internal_bus = InternalBus() self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.nodeStatusDB = None self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = create_view_changer(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def __init__(self): self.name = 'Node1' self.f = 1 self.replicas = [] self.viewNo = 0 self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.replicas = [ Replica(node=self, instId=0, isMaster=True), Replica(node=self, instId=1, isMaster=False), Replica(node=self, instId=2, isMaster=False), ] self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes)
def _init_common_managers(self): # Pool manager init self.node.poolManager = TxnPoolManager( self.node, self.node.poolLedger, self.node.states[POOL_LEDGER_ID], self.node.write_manager, self.node.ha, self.node.cliname, self.node.cliha) # Ledger manager init ledger_sync_order = self.node.ledger_ids self.node.ledgerManager = LedgerManager( self.node, postAllLedgersCaughtUp=self.node.allLedgersCaughtUp, preCatchupClbk=self.node.preLedgerCatchUp, postCatchupClbk=self.node.postLedgerCatchUp, ledger_sync_order=ledger_sync_order, metrics=self.node.metrics)
def test_missing_txn_request(ledger_no_genesis): """ Testing LedgerManager's `_missing_txns` """ ledger = ledger_no_genesis for i in range(20): txn = random_txn(i) ledger.add(txn) # Callbacks don't matter in this test ledger_info = LedgerInfo(0, ledger, *[None] * 6) assert ledger_info.catchupReplyTimer is None assert LedgerManager._missing_txns(ledger_info) == (False, 0) ledger_info.catchupReplyTimer = time.perf_counter() # Ledger is already ahead cp = ConsistencyProof(0, 1, 10, 1, 1, 'GJybBTHjzMzPWsE6n9qNQWAmhJP88dTcdbgkGLhYGFYn', 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(1, 15)] assert not LedgerManager._missing_txns(ledger_info)[0] # Ledger is behind but catchup replies present cp = ConsistencyProof(0, 1, 30, 1, 1, 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', 'EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 31)] assert not LedgerManager._missing_txns(ledger_info)[0] ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 35)] assert not LedgerManager._missing_txns(ledger_info)[0] # Ledger is behind cp = ConsistencyProof(0, 1, 30, 1, 1, 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', 'EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 26)] assert LedgerManager._missing_txns(ledger_info) == (True, 5) ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(26, 31)] assert LedgerManager._missing_txns(ledger_info) == (True, 5)
def test_missing_txn_request(ledger_no_genesis): """ Testing LedgerManager's `_missing_txns` """ ledger = ledger_no_genesis for i in range(20): txn = random_txn(i) ledger.add(txn) # Callbacks don't matter in this test ledger_info = LedgerInfo(0, ledger, *[None] * 6) assert ledger_info.catchupReplyTimer is None assert LedgerManager._missing_txns(ledger_info) == (False, 0) ledger_info.catchupReplyTimer = time.perf_counter() # Ledger is already ahead cp = ConsistencyProof(0, 1, 10, 1, 1, 'GJybBTHjzMzPWsE6n9qNQWAmhJP88dTcdbgkGLhYGFYn', 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(1, 15)] assert not LedgerManager._missing_txns(ledger_info)[0] # Ledger is behind but catchup replies present cp = ConsistencyProof(0, 1, 30, 1, 1, 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', 'EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 31)] assert not LedgerManager._missing_txns(ledger_info)[0] ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 35)] assert not LedgerManager._missing_txns(ledger_info)[0] # Ledger is behind cp = ConsistencyProof(0, 1, 30, 1, 1, 'Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC', 'EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC', []) ledger_info.catchUpTill = cp ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(21, 26)] assert LedgerManager._missing_txns(ledger_info) == (True, 5) ledger_info.receivedCatchUpReplies = [(i, {}) for i in range(26, 31)] assert LedgerManager._missing_txns(ledger_info) == (True, 5)
def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {}
class FakeNode: ledger_ids = [POOL_LEDGER_ID, CONFIG_LEDGER_ID, DOMAIN_LEDGER_ID] def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.internal_bus = InternalBus() self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.nodeStatusDB = None self.internal_bus = InternalBus() self.quorums = Quorums(self.totalNodes) self.nodestack = FakeSomething(connecteds=set(self.allNodeNames)) self.write_manager = FakeSomething() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config) } self.requiredNumberOfInstances = 2 self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = create_view_changer(self) self.primaries_selector = RoundRobinPrimariesSelector() self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True @property def viewNo(self): return None if self.view_changer is None else self.view_changer.view_no @property def ledger_summary(self): return [li.ledger_summary for li in self.ledgerManager.ledgerRegistry.values()] def get_name_by_rank(self, name, node_reg, node_ids): # This is used only for getting name of next primary, so # it just returns a constant return 'Node2' def primary_selected(self, instance_id): self._found = True def is_primary_found(self): return self._found @property def master_primary_name(self) -> Optional[str]: nm = self.replicas[0].primaryName if nm: return Replica.getNodeName(nm) @property def master_replica(self): return self.replicas[0] @property def is_synced(self): return self.mode >= Mode.synced def on_view_change_start(self): pass def start_catchup(self): pass def allLedgersCaughtUp(self): Node.allLedgersCaughtUp(self) def _clean_non_forwarded_ordered(self): return Node._clean_non_forwarded_ordered(self) def num_txns_caught_up_in_last_catchup(self): return Node.num_txns_caught_up_in_last_catchup(self) def set_view_change_status(self, value): return Node.set_view_change_status(self, value) def mark_request_as_executed(self, request): Node.mark_request_as_executed(self, request) def _clean_req_from_verified(self, request): pass def doneProcessingReq(self, key): pass def is_catchup_needed(self): return False def no_more_catchups_needed(self): pass def select_primaries(self): pass def utc_epoch(self): return get_utc_epoch() def get_validators(self): return [] def set_view_for_replicas(self, a): pass def get_primaries_for_current_view(self): # This is used only for getting name of next primary, so # it just returns a constant return ['Node2', 'Node3']
class Client(Motor, MessageProcessor, HasFileStorage, HasPoolManager, HasActionQueue): def __init__(self, name: str, nodeReg: Dict[str, HA]=None, ha: Union[HA, Tuple[str, int]]=None, basedirpath: str=None, config=None, sighex: str=None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath signer = Signer(sighex) sighex = signer.keyhex verkey = rawToFriendly(signer.verraw) self.name = name self.stackName = verkey cha = None # If client information already exists is RAET then use that if self.exists(self.stackName, basedirpath): cha = getHaFromLocalEstate(self.stackName, basedirpath) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger(0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setF() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auto=AutoMode.always) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info("Client {} initialized with the following node registry:" .format(self.name)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.name)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp)) def getReqRepStore(self): return ClientReqRepStoreFile(self.name, self.basedirpath) def getTxnLogStore(self): return ClientTxnLog(self.name, self.basedirpath) def __repr__(self): return self.name def postPoolLedgerCaughtUp(self): self.mode = Mode.discovered # For the scenario where client has already connected to nodes reading # the genesis pool transactions and that is enough if self.hasSufficientConnections: self.flushMsgsPendingConnection() def postTxnFromCatchupAddedToLedger(self, ledgerType: int, txn: Any): if ledgerType != 0: logger.error("{} got unknown ledger type {}". format(self, ledgerType)) return self.processPoolTxn(txn) # noinspection PyAttributeOutsideInit def setF(self): nodeCount = len(self.nodeReg) self.f = getMaxFailures(nodeCount) self.minNodesToConnect = self.f + 1 self.totalNodes = nodeCount @staticmethod def exists(name, basedirpath): return os.path.exists(basedirpath) and \ os.path.exists(os.path.join(basedirpath, name)) @property def nodeStackClass(self) -> NodeStack: return NodeStack def start(self, loop): oldstatus = self.status if oldstatus in Status.going(): logger.info("{} is already {}, so start has no effect". format(self, self.status.name)) else: super().start(loop) self.nodestack.start() self.nodestack.maintainConnections() if self._ledger: self.ledgerManager.setLedgerCanSync(0, True) self.mode = Mode.starting async def prod(self, limit) -> int: """ async function that returns the number of events :param limit: The number of messages to be processed :return: The number of events up to a prescribed `limit` """ s = 0 if self.isGoing(): s = await self.nodestack.service(limit) await self.nodestack.serviceLifecycle() self.nodestack.flushOutBoxes() s += self._serviceActions() # TODO: This if condition has to be removed. `_ledger` if once set wont # be reset ever so in `__init__` the `prod` method should be patched. if self._ledger: s += self.ledgerManager._serviceActions() return s def submitReqs(self, *reqs: Request) -> List[Request]: requests = [] for request in reqs: if self.mode == Mode.discovered and self.hasSufficientConnections: self.nodestack.send(request) self.expectingFor(request) else: logger.debug("{} pending request since in mode {} and " "connected to {} nodes". format(self, self.mode, self.nodestack.connecteds)) self.pendReqsTillConnection(request) requests.append(request) for r in requests: self.reqRepStore.addRequest(r) return requests def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None: """ Handles single message from a node, and appends it to a queue :param wrappedMsg: Reply received by the client from the node """ self.inBox.append(wrappedMsg) msg, frm = wrappedMsg # Do not print result of transaction type `POOL_LEDGER_TXNS` on the CLI ledgerTxnTypes = (POOL_LEDGER_TXNS, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REP) printOnCli = not excludeFromCli and msg.get(OP_FIELD_NAME) not \ in ledgerTxnTypes logger.debug("Client {} got msg from node {}: {}". format(self.name, frm, msg), extra={"cli": printOnCli}) if OP_FIELD_NAME in msg: if msg[OP_FIELD_NAME] in ledgerTxnTypes and self._ledger: op = msg.get(OP_FIELD_NAME, None) if not op: raise MissingNodeOp # TODO: Refactor this copying cls = TaggedTuples.get(op, None) t = copy.deepcopy(msg) t.pop(OP_FIELD_NAME, None) cMsg = cls(**t) if msg[OP_FIELD_NAME] == POOL_LEDGER_TXNS: self.poolTxnReceived(cMsg, frm) if msg[OP_FIELD_NAME] == LEDGER_STATUS: self.ledgerManager.processLedgerStatus(cMsg, frm) if msg[OP_FIELD_NAME] == CONSISTENCY_PROOF: self.ledgerManager.processConsistencyProof(cMsg, frm) if msg[OP_FIELD_NAME] == CATCHUP_REP: self.ledgerManager.processCatchupRep(cMsg, frm) elif msg[OP_FIELD_NAME] == REQACK: self.reqRepStore.addAck(msg, frm) self.gotExpected(msg, frm) elif msg[OP_FIELD_NAME] == REQNACK: self.reqRepStore.addNack(msg, frm) self.gotExpected(msg, frm) elif msg[OP_FIELD_NAME] == REPLY: result = msg[f.RESULT.nm] identifier = msg[f.RESULT.nm][f.IDENTIFIER.nm] reqId = msg[f.RESULT.nm][f.REQ_ID.nm] numReplies = self.reqRepStore.addReply(identifier, reqId, frm, result) self.gotExpected(msg, frm) self.postReplyRecvd(identifier, reqId, frm, result, numReplies) def postReplyRecvd(self, identifier, reqId, frm, result, numReplies): if not self.txnLog.hasTxn(identifier, reqId) and numReplies > self.f: replies = self.reqRepStore.getReplies(identifier, reqId).values() reply = checkIfMoreThanFSameItems(replies, self.f) if reply: self.txnLog.append(identifier, reqId, reply) return reply def _statusChanged(self, old, new): # do nothing for now pass def onStopping(self, *args, **kwargs): self.nodestack.nextCheck = 0 self.nodestack.stop() if self._ledger: self.ledgerManager.setLedgerState(0, LedgerState.not_synced) self.mode = None def getReply(self, identifier: str, reqId: int) -> Optional[Reply]: """ Accepts reply message from node if the reply is matching :param identifier: identifier of the entity making the request :param reqId: Request Id :return: Reply message only when valid and matching (None, NOT_FOUND) (None, UNCONFIRMED) f+1 not reached (reply, CONFIRMED) f+1 reached """ try: cons = self.hasConsensus(identifier, reqId) except KeyError: return None, "NOT_FOUND" if cons: return cons, "CONFIRMED" return None, "UNCONFIRMED" def getRepliesFromAllNodes(self, identifier: str, reqId: int): """ Accepts a request ID and return a list of results from all the nodes for that request :param identifier: identifier of the entity making the request :param reqId: Request ID :return: list of request results from all nodes """ return {frm: msg for msg, frm in self.inBox if msg[OP_FIELD_NAME] == REPLY and msg[f.RESULT.nm][f.REQ_ID.nm] == reqId and msg[f.RESULT.nm][f.IDENTIFIER.nm] == identifier} def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]: """ Accepts a request ID and returns True if consensus was reached for the request or else False :param identifier: identifier of the entity making the request :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(identifier, reqId) if not replies: raise KeyError('{}{}'.format(identifier, reqId)) # NOT_FOUND # Check if at least f+1 replies are received or not. if self.f + 1 > len(replies): return False # UNCONFIRMED else: onlyResults = {frm: reply["result"] for frm, reply in replies.items()} resultsList = list(onlyResults.values()) # if all the elements in the resultList are equal - consensus # is reached. if all(result == resultsList[0] for result in resultsList): return resultsList[0] # CONFIRMED else: logger.error( "Received a different result from at least one of the nodes..") return checkIfMoreThanFSameItems(resultsList, self.f) def showReplyDetails(self, identifier: str, reqId: int): """ Accepts a request ID and prints the reply details :param identifier: Client's identifier :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(identifier, reqId) replyInfo = "Node {} replied with result {}" if replies: for frm, reply in replies.items(): print(replyInfo.format(frm, reply['result'])) else: print("No replies received from Nodes!") def onConnsChanged(self, joined: Set[str], left: Set[str]): """ Modify the current status of the client based on the status of the connections changed. """ if self.isGoing(): if len(self.nodestack.conns) == len(self.nodeReg): self.status = Status.started elif len(self.nodestack.conns) >= self.minNodesToConnect: self.status = Status.started_hungry if self.hasSufficientConnections and self.mode == Mode.discovered: self.flushMsgsPendingConnection() if self._ledger: for n in joined: self.sendLedgerStatus(n) def replyIfConsensus(self, identifier, reqId: int): replies, errors = self.reqRepStore.getAllReplies(identifier, reqId) r = list(replies.values())[0] if len(replies) > self.f else None e = list(errors.values())[0] if len(errors) > self.f else None return r, e @property def hasSufficientConnections(self): return len(self.nodestack.conns) >= self.minNodesToConnect def hasMadeRequest(self, identifier, reqId: int): return self.reqRepStore.hasRequest(identifier, reqId) def isRequestSuccessful(self, identifier, reqId): acks = self.reqRepStore.getAcks(identifier, reqId) nacks = self.reqRepStore.getNacks(identifier, reqId) f = getMaxFailures(len(self.nodeReg)) if len(acks) > f: return True, "Done" elif len(nacks) > f: # TODO: What if the the nacks were different from each node? return False, list(nacks.values())[0] else: return None def pendReqsTillConnection(self, request, signer=None): """ Enqueue requests that need to be submitted until the client has sufficient connections to nodes :return: """ self.reqsPendingConnection.append((request, signer)) logger.debug("{} enqueuing request since not enough connections " "with nodes: {}".format(self, request)) def flushMsgsPendingConnection(self): queueSize = len(self.reqsPendingConnection) if queueSize > 0: logger.debug("Flushing pending message queue of size {}" .format(queueSize)) while self.reqsPendingConnection: req, signer = self.reqsPendingConnection.popleft() self.nodestack.send(req, signer=signer) def expectingFor(self, request: Request, nodes: Optional[Set[str]]=None): nodes = nodes or {r.name for r in self.nodestack.remotes.values() if self.nodestack.isRemoteConnected(r)} now = time.perf_counter() self.expectingAcksFor[request.key] = (nodes, now, 0) self.expectingRepliesFor[request.key] = (copy.copy(nodes), now, 0) self.startRepeating(self.retryForExpected, self.config.CLIENT_REQACK_TIMEOUT) def gotExpected(self, msg, frm): if msg[OP_FIELD_NAME] == REQACK: container = msg colls = (self.expectingAcksFor, ) elif msg[OP_FIELD_NAME] == REPLY: container = msg[f.RESULT.nm] # If an REQACK sent by node was lost, the request when sent again # would fetch the reply or the client might just lose REQACK and not # REPLY so when REPLY received, request does not need to be resent colls = (self.expectingAcksFor, self.expectingRepliesFor) elif msg[OP_FIELD_NAME] == REQNACK: container = msg colls = (self.expectingAcksFor, self.expectingRepliesFor) else: raise RuntimeError("{} cannot retry {}".format(self, msg)) idr = container.get(f.IDENTIFIER.nm) reqId = container.get(f.REQ_ID.nm) key = (idr, reqId) for coll in colls: if key in coll: if frm in coll[key][0]: coll[key][0].remove(frm) if not coll[key][0]: coll.pop(key) if not (self.expectingAcksFor or self.expectingRepliesFor): self.stopRepeating(self.retryForExpected, strict=False) def retryForExpected(self): now = time.perf_counter() keys = {} nodesNotSendingAck = set() # Collect nodes which did not send REQACK clearKeys = [] for reqKey, (expectedFrom, lastTried, retries) in \ self.expectingAcksFor.items(): if now > (lastTried + self.config.CLIENT_REQACK_TIMEOUT): if retries < self.config.CLIENT_MAX_RETRY_ACK: if reqKey not in keys: keys[reqKey] = set() keys[reqKey].update(expectedFrom) nodesNotSendingAck.update(expectedFrom) else: clearKeys.append(reqKey) for k in clearKeys: self.expectingAcksFor.pop(k) # Collect nodes which did not send REPLY clearKeys = [] for reqKey, (expectedFrom, lastTried, retries) in \ self.expectingRepliesFor.items(): if now > (lastTried + self.config.CLIENT_REPLY_TIMEOUT): if retries < self.config.CLIENT_MAX_RETRY_REPLY: if reqKey not in keys: keys[reqKey] = set() keys[reqKey].update(expectedFrom) else: clearKeys.append(reqKey) for k in clearKeys: self.expectingRepliesFor.pop(k) for nm in nodesNotSendingAck: try: remote = self.nodestack.getRemote(nm) except RemoteNotFound: logger.warn('{} could not find remote {}'.format(self, nm)) continue logger.debug('Remote {} of {} being joined since REQACK for not ' 'received for request'.format(remote, self)) self.nodestack.join(remote.uid, cascade=True) if keys: # Need a delay in case connection has to be established with some # nodes, a better way is not to assume the delay value but only # send requests once the connection is established. Also it is # assumed that connection is not established if a node not sending # REQACK/REQNACK/REPLY, but a little better way is to compare the # value in stats of the stack and look for changes in count of # `message_reject_rx` but that is not very helpful either since # it does not record which node rejected delay = 3 if nodesNotSendingAck else 0 self._schedule(partial(self.resendRequests, keys), delay) def resendRequests(self, keys): for key, nodes in keys.items(): if nodes: request = self.reqRepStore.getRequest(*key) logger.debug('{} resending request {} to {}'. format(self, request, nodes)) self.sendToNodes(request, nodes) now = time.perf_counter() if key in self.expectingAcksFor: _, _, c = self.expectingAcksFor[key] self.expectingAcksFor[key] = (nodes, now, c + 1) if key in self.expectingRepliesFor: _, _, c = self.expectingRepliesFor[key] self.expectingRepliesFor[key] = (nodes, now, c + 1) def sendLedgerStatus(self, nodeName: str): ledgerStatus = LedgerStatus(0, self.ledger.size, self.ledger.root_hash) rid = self.nodestack.getRemote(nodeName).uid self.nodestack.send(ledgerStatus, rid) def send(self, msg: Any, *rids: Iterable[int], signer: Signer = None): self.nodestack.send(msg, *rids, signer=signer) def sendToNodes(self, msg: Any, names: Iterable[str]): rids = [rid for rid, r in self.nodestack.remotes.items() if r.name in names] self.nodestack.send(msg, *rids) @staticmethod def verifyMerkleProof(*replies: Tuple[Reply]) -> bool: """ Verifies the correctness of the merkle proof provided in the reply from the node. Returns True if verified to be correct, throws an exception otherwise. :param replies: One or more replies for which Merkle Proofs have to be verified :raises ProofError: The proof is invalid :return: True """ verifier = MerkleVerifier() fields = getTxnOrderedFields() serializer = CompactSerializer(fields=fields) for r in replies: seqNo = r[f.RESULT.nm][F.seqNo.name] rootHash = base64.b64decode( r[f.RESULT.nm][F.rootHash.name].encode()) auditPath = [base64.b64decode( a.encode()) for a in r[f.RESULT.nm][F.auditPath.name]] filtered = ((k, v) for (k, v) in r[f.RESULT.nm].iteritems() if k not in [F.auditPath.name, F.seqNo.name, F.rootHash.name]) result = serializer.serialize(dict(filtered)) verifier.verify_leaf_inclusion(result, seqNo - 1, auditPath, STH(tree_size=seqNo, sha256_root_hash=rootHash)) return True
class FakeNode: ledger_ids = [0] def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = [] self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() @property def viewNo(self): return None if self.view_changer is None else self.view_changer.view_no @property def ledger_summary(self): return [li.ledger_summary for li in self.ledgerManager.ledgerRegistry.values()] def get_name_by_rank(self, name, nodeReg=None): # This is used only for getting name of next primary, so # it just returns a constant return 'Node2' def primary_selected(self, instance_id): self._found = True def is_primary_found(self): return self._found @property def master_primary_name(self) -> Optional[str]: nm = self.replicas[0].primaryName if nm: return Replica.getNodeName(nm) @property def master_replica(self): return self.replicas[0] @property def is_synced(self): return self.mode >= Mode.synced def on_view_change_start(self): pass def start_catchup(self): pass
def __init__(self, name: str, nodeReg: Dict[str, HA]=None, ha: Union[HA, Tuple[str, int]]=None, basedirpath: str=None, config=None, sighex: str=None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath signer = Signer(sighex) sighex = signer.keyhex verkey = rawToFriendly(signer.verraw) self.name = name self.stackName = verkey cha = None # If client information already exists is RAET then use that if self.exists(self.stackName, basedirpath): cha = getHaFromLocalEstate(self.stackName, basedirpath) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger(0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setF() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auto=AutoMode.always) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info("Client {} initialized with the following node registry:" .format(self.name)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.name)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp))
def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, config=None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port :param lastReqId: Request Id of the last request sent by client """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath cha = None # If client information already exists is RAET then use that if self.exists(name, basedirpath): logger.debug("Client {} ignoring given ha".format(ha)) cha = getHaFromLocalEstate(name, basedirpath) if cha: cha = HA(*cha) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.name = name self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( 0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered self.setF() stackargs = dict( name=name, ha=cha, main=False, # stops incoming vacuous joins auto=AutoMode.always) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg) self.nodestack.onConnsChanged = self.onConnsChanged logger.info( "Client {} initialized with the following node registry:".format( name)) lengths = [ max(x) for x in zip(*[(len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()]) ] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp))
class Client(Motor, MessageProcessor, HasFileStorage, HasPoolManager): def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, config=None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port :param lastReqId: Request Id of the last request sent by client """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath cha = None # If client information already exists is RAET then use that if self.exists(name, basedirpath): logger.debug("Client {} ignoring given ha".format(ha)) cha = getHaFromLocalEstate(name, basedirpath) if cha: cha = HA(*cha) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.name = name self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( 0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered self.setF() stackargs = dict( name=name, ha=cha, main=False, # stops incoming vacuous joins auto=AutoMode.always) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg) self.nodestack.onConnsChanged = self.onConnsChanged logger.info( "Client {} initialized with the following node registry:".format( name)) lengths = [ max(x) for x in zip(*[(len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()]) ] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp)) def getReqRepStore(self): return ClientReqRepStoreFile(self.name, self.basedirpath) def getTxnLogStore(self): return ClientTxnLog(self.name, self.basedirpath) def __repr__(self): return self.name def postPoolLedgerCaughtUp(self): self.mode = Mode.discovered # For the scenario where client has already connected to nodes reading # the genesis pool transactions and that is enough if self.hasSufficientConnections: self.flushMsgsPendingConnection() def postTxnFromCatchupAddedToLedger(self, ledgerType: int, txn: Any): if ledgerType != 0: logger.error("{} got unknown ledger type {}".format( self, ledgerType)) return self.processPoolTxn(txn) # noinspection PyAttributeOutsideInit def setF(self): nodeCount = len(self.nodeReg) self.f = getMaxFailures(nodeCount) self.minNodesToConnect = self.f + 1 self.totalNodes = nodeCount @staticmethod def exists(name, basedirpath): return os.path.exists(basedirpath) and \ os.path.exists(os.path.join(basedirpath, name)) @property def nodeStackClass(self) -> NodeStack: return NodeStack def start(self, loop): oldstatus = self.status if oldstatus in Status.going(): logger.info("{} is already {}, so start has no effect".format( self, self.status.name)) else: super().start(loop) self.nodestack.start() self.nodestack.maintainConnections() if self._ledger: self.ledgerManager.setLedgerCanSync(0, True) self.mode = Mode.starting async def prod(self, limit) -> int: """ async function that returns the number of events :param limit: The number of messages to be processed :return: The number of events up to a prescribed `limit` """ s = 0 if self.isGoing(): s = await self.nodestack.service(limit) await self.nodestack.serviceLifecycle() self.nodestack.flushOutBoxes() # TODO: This if condition has to be removed. `_ledger` if once set wont # be reset ever so in `__init__` the `prod` method should be patched. if self._ledger: s += self.ledgerManager._serviceActions() return s def createRequest(self, operation: Mapping, identifier: str = None) -> Request: """ Client creates request which include requested operation and request Id :param operation: requested operation :return: New client request """ request = Request(identifier=identifier or self.defaultIdentifier, operation=operation) # DEPR # self.setReqId(request) return request def submitReqs(self, *reqs: Request) -> List[Request]: requests = [] for request in reqs: # DEPR # self.setReqId(request) if self.mode == Mode.discovered and self.hasSufficientConnections: self.nodestack.send(request) else: self.pendReqsTillConnection(request) requests.append(request) for r in requests: self.reqRepStore.addRequest(r) return requests def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None: """ Handles single message from a node, and appends it to a queue :param wrappedMsg: Reply received by the client from the node """ self.inBox.append(wrappedMsg) msg, frm = wrappedMsg # Do not print result of transaction type `POOL_LEDGER_TXNS` on the CLI txnTypes = (POOL_LEDGER_TXNS, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REP) printOnCli = not excludeFromCli and msg.get(OP_FIELD_NAME) not \ in txnTypes logger.debug("Client {} got msg from node {}: {}".format( self.name, frm, msg), extra={"cli": printOnCli}) if OP_FIELD_NAME in msg: if msg[OP_FIELD_NAME] in txnTypes and self._ledger: op = msg.get(OP_FIELD_NAME, None) if not op: raise MissingNodeOp # TODO: Refactor this copying cls = TaggedTuples.get(op, None) t = copy.deepcopy(msg) t.pop(OP_FIELD_NAME, None) cMsg = cls(**t) if msg[OP_FIELD_NAME] == POOL_LEDGER_TXNS: self.poolTxnReceived(cMsg, frm) if msg[OP_FIELD_NAME] == LEDGER_STATUS: self.ledgerManager.processLedgerStatus(cMsg, frm) if msg[OP_FIELD_NAME] == CONSISTENCY_PROOF: self.ledgerManager.processConsistencyProof(cMsg, frm) if msg[OP_FIELD_NAME] == CATCHUP_REP: self.ledgerManager.processCatchupRep(cMsg, frm) elif msg[OP_FIELD_NAME] == REQACK: self.reqRepStore.addAck(msg, frm) elif msg[OP_FIELD_NAME] == REQNACK: self.reqRepStore.addNack(msg, frm) elif msg[OP_FIELD_NAME] == REPLY: result = msg[f.RESULT.nm] reqId = msg[f.RESULT.nm][f.REQ_ID.nm] numReplies = self.reqRepStore.addReply(reqId, frm, result) self.postReplyRecvd(reqId, frm, result, numReplies) def postReplyRecvd(self, reqId, frm, result, numReplies): if not self.txnLog.hasTxnWithReqId(reqId) and numReplies > self.f: replies = self.reqRepStore.getReplies(reqId).values() reply = checkIfMoreThanFSameItems(replies, self.f) if reply: self.txnLog.append(reqId, reply) return reply def _statusChanged(self, old, new): # do nothing for now pass def onStopping(self, *args, **kwargs): self.nodestack.nextCheck = 0 self.nodestack.stop() if self._ledger: self.ledgerManager.setLedgerState(0, LedgerState.not_synced) self.mode = None def getReply(self, reqId: int) -> Optional[Reply]: """ Accepts reply message from node if the reply is matching :param reqId: Request Id :return: Reply message only when valid and matching (None, NOT_FOUND) (None, UNCONFIRMED) f+1 not reached (reply, CONFIRMED) f+1 reached """ try: cons = self.hasConsensus(reqId) except KeyError: return None, "NOT_FOUND" if cons: return cons, "CONFIRMED" return None, "UNCONFIRMED" def getRepliesFromAllNodes(self, reqId: int): """ Accepts a request ID and return a list of results from all the nodes for that request :param reqId: Request ID :return: list of request results from all nodes """ return { frm: msg for msg, frm in self.inBox if msg[OP_FIELD_NAME] == REPLY and msg[f.RESULT.nm][f.REQ_ID.nm] == reqId } def hasConsensus(self, reqId: int) -> Optional[str]: """ Accepts a request ID and returns True if consensus was reached for the request or else False :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(reqId) if not replies: raise KeyError(reqId) # NOT_FOUND # Check if at least f+1 replies are received or not. if self.f + 1 > len(replies): return False # UNCONFIRMED else: onlyResults = { frm: reply["result"] for frm, reply in replies.items() } resultsList = list(onlyResults.values()) # if all the elements in the resultList are equal - consensus # is reached. if all(result == resultsList[0] for result in resultsList): return resultsList[0] # CONFIRMED else: logger.error( "Received a different result from at least one of the nodes.." ) return checkIfMoreThanFSameItems(resultsList, self.f) def showReplyDetails(self, reqId: int): """ Accepts a request ID and prints the reply details :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(reqId) replyInfo = "Node {} replied with result {}" if replies: for frm, reply in replies.items(): print(replyInfo.format(frm, reply['result'])) else: print("No replies received from Nodes!") def onConnsChanged(self, joined: Set[str], left: Set[str]): """ Modify the current status of the client based on the status of the connections changed. """ if self.isGoing(): if len(self.nodestack.conns) == len(self.nodeReg): self.status = Status.started elif len(self.nodestack.conns) >= self.minNodesToConnect: self.status = Status.started_hungry if self.hasSufficientConnections and self.mode == Mode.discovered: self.flushMsgsPendingConnection() if self._ledger: for n in joined: self.sendLedgerStatus(n) def replyIfConsensus(self, reqId: int): replies, errors = self.reqRepStore.getAllReplies(reqId) r = list(replies.values())[0] if len(replies) > self.f else None e = list(errors.values())[0] if len(errors) > self.f else None return r, e @property def hasSufficientConnections(self): return len(self.nodestack.conns) >= self.minNodesToConnect def hasMadeRequest(self, reqId: int): return self.reqRepStore.hasRequest(reqId) def isRequestSuccessful(self, reqId): acks = self.reqRepStore.getAcks(reqId) nacks = self.reqRepStore.getNacks(reqId) f = getMaxFailures(len(self.nodeReg)) if len(acks) > f: return True, "Done" elif len(nacks) > f: # TODO: What if the the nacks were different from each node? return False, list(nacks.values())[0] else: return None def pendReqsTillConnection(self, request, signer=None): """ Enqueue requests that need to be submitted until the client has sufficient connections to nodes :return: """ self.reqsPendingConnection.append((request, signer)) logger.debug("Enqueuing request since not enough connections " "with nodes: {}".format(request)) def flushMsgsPendingConnection(self): queueSize = len(self.reqsPendingConnection) if queueSize > 0: logger.debug( "Flushing pending message queue of size {}".format(queueSize)) while self.reqsPendingConnection: req, signer = self.reqsPendingConnection.popleft() self.nodestack.send(req, signer=signer) def sendLedgerStatus(self, nodeName: str): ledgerStatus = LedgerStatus(0, self.ledger.size, self.ledger.root_hash) rid = self.nodestack.getRemote(nodeName).uid self.nodestack.send(ledgerStatus, rid) def send(self, msg: Any, *rids: Iterable[int], signer: Signer = None): self.nodestack.send(msg, *rids, signer=signer) @staticmethod def verifyMerkleProof(*replies: Tuple[Reply]) -> bool: """ Verifies the correctness of the merkle proof provided in the reply from the node. Returns True if verified to be correct, throws an exception otherwise. :param replies: One or more replies for which Merkle Proofs have to be verified :raises ProofError: The proof is invalid :return: True """ verifier = MerkleVerifier() fields = getTxnOrderedFields() serializer = CompactSerializer(fields=fields) for r in replies: seqNo = r[f.RESULT.nm][F.seqNo.name] rootHash = base64.b64decode( r[f.RESULT.nm][F.rootHash.name].encode()) auditPath = [ base64.b64decode(a.encode()) for a in r[f.RESULT.nm][F.auditPath.name] ] filtered = ( (k, v) for (k, v) in r[f.RESULT.nm].iteritems() if k not in [F.auditPath.name, F.seqNo.name, F.rootHash.name]) result = serializer.serialize(dict(filtered)) verifier.verify_leaf_inclusion( result, seqNo - 1, auditPath, STH(tree_size=seqNo, sha256_root_hash=rootHash)) return True
class FakeNode: ledger_ids = [POOL_LEDGER_ID, CONFIG_LEDGER_ID, DOMAIN_LEDGER_ID] def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} @property def viewNo(self): return None if self.view_changer is None else self.view_changer.view_no @property def ledger_summary(self): return [li.ledger_summary for li in self.ledgerManager.ledgerRegistry.values()] def get_name_by_rank(self, name, nodeReg=None): # This is used only for getting name of next primary, so # it just returns a constant return 'Node2' def primary_selected(self, instance_id): self._found = True def is_primary_found(self): return self._found @property def master_primary_name(self) -> Optional[str]: nm = self.replicas[0].primaryName if nm: return Replica.getNodeName(nm) @property def master_replica(self): return self.replicas[0] @property def is_synced(self): return self.mode >= Mode.synced def on_view_change_start(self): pass def start_catchup(self): pass def allLedgersCaughtUp(self): Node.allLedgersCaughtUp(self) def _clean_non_forwarded_ordered(self): return Node._clean_non_forwarded_ordered(self) def num_txns_caught_up_in_last_catchup(self): return Node.num_txns_caught_up_in_last_catchup(self) def mark_request_as_executed(self, request): Node.mark_request_as_executed(self, request) def _clean_req_from_verified(self, request): pass def doneProcessingReq(self, key): pass def processStashedOrderedReqs(self): pass def is_catchup_needed(self): return False def no_more_catchups_needed(self): pass def select_primaries(self): pass
def ledger_manager(): fakeNode = FakeSomething(timer=None, allNodeNames=set('Node1')) lm = LedgerManager(fakeNode) setattr(fakeNode, 'ledgerManager', lm) return lm
def test_ledger_status_quorum(): N = 10 f = getMaxFailures(N) assert not LedgerManager.has_ledger_status_quorum(f + 1, N) assert LedgerManager.has_ledger_status_quorum(N - f - 1, N)
class FakeNode: ledger_ids = [POOL_LEDGER_ID, CONFIG_LEDGER_ID, DOMAIN_LEDGER_ID] def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} @property def viewNo(self): return None if self.view_changer is None else self.view_changer.view_no @property def ledger_summary(self): return [ li.ledger_summary for li in self.ledgerManager.ledgerRegistry.values() ] def get_name_by_rank(self, name, nodeReg=None): # This is used only for getting name of next primary, so # it just returns a constant return 'Node2' def primary_selected(self, instance_id): self._found = True def is_primary_found(self): return self._found @property def master_primary_name(self) -> Optional[str]: nm = self.replicas[0].primaryName if nm: return Replica.getNodeName(nm) @property def master_replica(self): return self.replicas[0] @property def is_synced(self): return self.mode >= Mode.synced def on_view_change_start(self): pass def start_catchup(self): pass def allLedgersCaughtUp(self): Node.allLedgersCaughtUp(self) def _clean_non_forwarded_ordered(self): return Node._clean_non_forwarded_ordered(self) def num_txns_caught_up_in_last_catchup(self): return Node.num_txns_caught_up_in_last_catchup(self) def mark_request_as_executed(self, request): Node.mark_request_as_executed(self, request) def _clean_req_from_verified(self, request): pass def doneProcessingReq(self, key): pass def processStashedOrderedReqs(self): pass def is_catchup_needed(self): return False def no_more_catchups_needed(self): pass def select_primaries(self): pass def utc_epoch(self): return get_utc_epoch()
class Client(Motor, MessageProcessor, HasFileStorage, HasPoolManager, HasActionQueue): def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, genesis_dir: str = None, ledger_dir: str = None, keys_dir: str = None, plugins_dir: str = None, config=None, sighex: str = None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() dataDir = self.config.clientDataDir or "data/clients" self.basedirpath = basedirpath or self.config.CLI_BASE_DIR self.basedirpath = os.path.expanduser(self.basedirpath) signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName or 'Client~' + str(id(self)) self.genesis_dir = genesis_dir or self.basedirpath self.ledger_dir = ledger_dir or os.path.join(self.basedirpath, dataDir, self.name) self.plugins_dir = plugins_dir or self.basedirpath _keys_dir = keys_dir or self.basedirpath self.keys_dir = os.path.join(_keys_dir, "keys") cha = None if self.exists(self.stackName, self.keys_dir): cha = self.nodeStackClass.getHaFromLocal( self.stackName, self.keys_dir) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() HasFileStorage.__init__(self, self.ledger_dir) # TODO: Find a proper name self.alias = name if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( POOL_LEDGER_ID, self.ledger, preCatchupStartClbk=self.prePoolLedgerCatchup, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setPoolParams() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auth_mode=AuthMode.ALLOW_ANY.value) stackargs['basedirpath'] = self.keys_dir self.created = time.perf_counter() # noinspection PyCallingNonCallable # TODO I think this is a bug here, sighex is getting passed in the seed # parameter self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info( "Client {} initialized with the following node registry:".format( self.alias)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} self._observers = {} # type Dict[str, Callable] self._observerSet = set() # makes it easier to guard against duplicates plugins_to_load = self.config.PluginsToLoad if hasattr(self.config, "PluginsToLoad") else None tp = loadPlugins(self.plugins_dir, plugins_to_load) logger.debug("total plugins loaded in client: {}".format(tp)) self._multi_sig_verifier = self._create_multi_sig_verifier() self._read_only_requests = set() @lazy_field def _bls_register(self): return BlsKeyRegisterPoolLedger(self.ledger) def _create_multi_sig_verifier(self) -> BlsCryptoVerifier: verifier = create_default_bls_crypto_factory() \ .create_bls_crypto_verifier() return verifier def getReqRepStore(self): return ClientReqRepStoreFile(self.ledger_dir) def getTxnLogStore(self): return ClientTxnLog(self.ledger_dir) def __repr__(self): return self.name def prePoolLedgerCatchup(self, **kwargs): self.mode = Mode.discovering def postPoolLedgerCaughtUp(self): self.mode = Mode.discovered # For the scenario where client has already connected to nodes reading # the genesis pool transactions and that is enough self.flushMsgsPendingConnection() def postTxnFromCatchupAddedToLedger(self, ledgerType: int, txn: Any): if ledgerType != 0: logger.error("{} got unknown ledger type {}". format(self, ledgerType)) return self.processPoolTxn(txn) # noinspection PyAttributeOutsideInit def setPoolParams(self): nodeCount = len(self.nodeReg) self.f = getMaxFailures(nodeCount) self.minNodesToConnect = self.f + 1 self.totalNodes = nodeCount self.quorums = Quorums(nodeCount) logger.info( "{} updated its pool parameters: f {}, totalNodes {}," "minNodesToConnect {}, quorums {}".format( self.alias, self.f, self.totalNodes, self.minNodesToConnect, self.quorums)) @staticmethod def exists(name, base_dir): return os.path.exists(base_dir) and \ os.path.exists(os.path.join(base_dir, name)) @property def nodeStackClass(self) -> NetworkInterface: return nodeStackClass def start(self, loop): oldstatus = self.status if oldstatus in Status.going(): logger.info("{} is already {}, so start has no effect". format(self.alias, self.status.name)) else: super().start(loop) self.nodestack.start() self.nodestack.maintainConnections(force=True) if self.ledger: self.ledgerManager.setLedgerCanSync(POOL_LEDGER_ID, True) self.mode = Mode.starting async def prod(self, limit) -> int: """ async function that returns the number of events :param limit: The number of messages to be processed :return: The number of events up to a prescribed `limit` """ s = 0 if self.isGoing(): s = await self.nodestack.service(limit) self.nodestack.serviceLifecycle() self.nodestack.flushOutBoxes() s += self._serviceActions() # TODO: This if condition has to be removed. `_ledger` if once set wont # be reset ever so in `__init__` the `prod` method should be patched. if self.ledger: s += self.ledgerManager._serviceActions() return s def submitReqs(self, *reqs: Request) -> Tuple[List[Request], List[str]]: requests = [] errs = [] for request in reqs: is_read_only = request.txn_type in self._read_only_requests if self.can_send_request(request): recipients = self._connected_node_names if is_read_only and len(recipients) > 1: recipients = random.sample(list(recipients), 1) logger.debug('Client {} sending request {} to recipients {}' .format(self, request, recipients)) stat, err_msg = self.sendToNodes(request, names=recipients) if stat: self._expect_replies(request, recipients) else: errs.append(err_msg) logger.debug( 'Client {} request failed {}'.format(self, err_msg)) continue else: logger.debug( "{} pending request since in mode {} and " "connected to {} nodes".format( self, self.mode, self.nodestack.connecteds)) self.pendReqsTillConnection(request) requests.append(request) for r in requests: self.reqRepStore.addRequest(r) return requests, errs def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None: """ Handles single message from a node, and appends it to a queue :param wrappedMsg: Reply received by the client from the node """ self.inBox.append(wrappedMsg) msg, frm = wrappedMsg # Do not print result of transaction type `POOL_LEDGER_TXNS` on the CLI ledgerTxnTypes = (POOL_LEDGER_TXNS, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REP) printOnCli = not excludeFromCli and msg.get(OP_FIELD_NAME) not \ in ledgerTxnTypes logger.info("Client {} got msg from node {}: {}". format(self.name, frm, msg), extra={"cli": printOnCli}) if OP_FIELD_NAME in msg: if msg[OP_FIELD_NAME] in ledgerTxnTypes and self.ledger: cMsg = node_message_factory.get_instance(**msg) if msg[OP_FIELD_NAME] == POOL_LEDGER_TXNS: self.poolTxnReceived(cMsg, frm) if msg[OP_FIELD_NAME] == LEDGER_STATUS: self.ledgerManager.processLedgerStatus(cMsg, frm) if msg[OP_FIELD_NAME] == CONSISTENCY_PROOF: self.ledgerManager.processConsistencyProof(cMsg, frm) if msg[OP_FIELD_NAME] == CATCHUP_REP: self.ledgerManager.processCatchupRep(cMsg, frm) elif msg[OP_FIELD_NAME] == REQACK: self.reqRepStore.addAck(msg, frm) self._got_expected(msg, frm) elif msg[OP_FIELD_NAME] == REQNACK: self.reqRepStore.addNack(msg, frm) self._got_expected(msg, frm) elif msg[OP_FIELD_NAME] == REJECT: self.reqRepStore.addReject(msg, frm) self._got_expected(msg, frm) elif msg[OP_FIELD_NAME] == REPLY: result = msg[f.RESULT.nm] identifier = get_reply_itentifier(result) reqId = get_reply_reqId(result) numReplies = self.reqRepStore.addReply(identifier, reqId, frm, result) self._got_expected(msg, frm) self.postReplyRecvd(identifier, reqId, frm, result, numReplies) def postReplyRecvd(self, identifier, reqId, frm, result, numReplies): if not self.txnLog.hasTxn(identifier, reqId): reply, _ = self.getReply(identifier, reqId) if reply: self.txnLog.append(identifier, reqId, reply) for name in self._observers: try: self._observers[name](name, reqId, frm, result, numReplies) except Exception as ex: # TODO: All errors should not be shown on CLI, or maybe we # show errors with different color according to the # severity. Like an error occurring due to node sending # a malformed message should not result in an error message # being shown on the cli since the clients would anyway # collect enough replies from other nodes. logger.debug("Observer threw an exception", exc_info=ex) return reply # Reply is not verified key = (identifier, reqId) if key not in self.expectingRepliesFor and numReplies == 1: # only one node was asked, but its reply cannot be confirmed, # so ask other nodes recipients = self._connected_node_names.difference({frm}) self.resendRequests({ (identifier, reqId): recipients }, force_expect=True) def _statusChanged(self, old, new): # do nothing for now pass def onStopping(self, *args, **kwargs): logger.debug('Stopping client {}'.format(self)) self.nodestack.nextCheck = 0 self.nodestack.stop() if self.ledger: self.ledgerManager.setLedgerState( POOL_LEDGER_ID, LedgerState.not_synced) self.mode = None self.ledger.stop() if self.hashStore and not self.hashStore.closed: self.hashStore.close() self.txnLog.close() def getReply(self, identifier: str, reqId: int) -> Optional: """ Accepts reply message from node if the reply is matching :param identifier: identifier of the entity making the request :param reqId: Request Id :return: Reply message only when valid and matching (None, NOT_FOUND) (None, UNCONFIRMED) f+1 not reached (reply, CONFIRMED) f+1 reached """ try: cons = self.hasConsensus(identifier, reqId) except KeyError: return None, "NOT_FOUND" if cons: return cons, "CONFIRMED" return None, "UNCONFIRMED" def getRepliesFromAllNodes(self, identifier: str, reqId: int): """ Accepts a request ID and return a list of results from all the nodes for that request :param identifier: identifier of the entity making the request :param reqId: Request ID :return: list of request results from all nodes """ return {frm: msg for msg, frm in self.inBox if msg[OP_FIELD_NAME] == REPLY and get_reply_reqId(msg[f.RESULT.nm]) == reqId and get_reply_itentifier(msg[f.RESULT.nm]) == identifier} def hasConsensus(self, identifier: str, reqId: int) -> Optional[Reply]: """ Accepts a request ID and returns reply for it if quorum achieved or there is a state proof for it. :param identifier: identifier of the entity making the request :param reqId: Request ID """ full_req_id = '({}:{})'.format(identifier, reqId) replies = self.getRepliesFromAllNodes(identifier, reqId) if not replies: raise KeyError(full_req_id) proved_reply = self.take_one_proved(replies, full_req_id) if proved_reply: logger.debug("Found proved reply for {}".format(full_req_id)) return proved_reply quorumed_reply = self.take_one_quorumed(replies, full_req_id) if quorumed_reply: logger.debug("Reply quorum for {} achieved" .format(full_req_id)) return quorumed_reply def take_one_quorumed(self, replies, full_req_id): """ Checks whether there is sufficint number of equal replies from different nodes. It uses following logic: 1. Check that there are sufficient replies received at all. If not - return None. 2. Check that all these replies are equal. If yes - return one of them. 3. Check that there is a group of equal replies which is large enough. If yes - return one reply from this group. 4. Return None """ if not self.quorums.reply.is_reached(len(replies)): return None # excluding state proofs from check since they can be different def without_state_proof(result): if STATE_PROOF in result: result.pop(STATE_PROOF) return result results = [without_state_proof(reply["result"]) for reply in replies.values()] first = results[0] if all(result == first for result in results): return first logger.debug("Received a different result from " "at least one node for {}" .format(full_req_id)) result, freq = mostCommonElement(results) if not self.quorums.reply.is_reached(freq): return None return result def take_one_proved(self, replies, full_req_id): """ Returns one reply with valid state proof """ for sender, reply in replies.items(): result = reply['result'] if STATE_PROOF not in result or result[STATE_PROOF] is None: logger.debug("There is no state proof in " "reply for {} from {}" .format(full_req_id, sender)) continue if not self.validate_multi_signature(result[STATE_PROOF]): logger.debug("{} got reply for {} with bad " "multi signature from {}" .format(self.name, full_req_id, sender)) # TODO: do something with this node continue if not self.validate_proof(result): logger.debug("{} got reply for {} with invalid " "state proof from {}" .format(self.name, full_req_id, sender)) # TODO: do something with this node continue return result def validate_multi_signature(self, state_proof): """ Validates multi signature """ multi_signature = state_proof[MULTI_SIGNATURE] if not multi_signature: logger.debug("There is a state proof, but no multi signature") return False participants = multi_signature[MULTI_SIGNATURE_PARTICIPANTS] signature = multi_signature[MULTI_SIGNATURE_SIGNATURE] value = MultiSignatureValue( **(multi_signature[MULTI_SIGNATURE_VALUE]) ).as_single_value() if not self.quorums.bls_signatures.is_reached(len(participants)): logger.debug("There is not enough participants of " "multi-signature") return False public_keys = [] for node_name in participants: key = self._bls_register.get_key_by_name(node_name) if key is None: logger.debug("There is no bls key for node {}" .format(node_name)) return False public_keys.append(key) return self._multi_sig_verifier.verify_multi_sig(signature, value, public_keys) def validate_proof(self, result): """ Validates state proof """ state_root_hash = result[STATE_PROOF]['root_hash'] state_root_hash = state_roots_serializer.deserialize(state_root_hash) proof_nodes = result[STATE_PROOF]['proof_nodes'] if isinstance(proof_nodes, str): proof_nodes = proof_nodes.encode() proof_nodes = proof_nodes_serializer.deserialize(proof_nodes) key, value = self.prepare_for_state(result) valid = PruningState.verify_state_proof(state_root_hash, key, value, proof_nodes, serialized=True) return valid def prepare_for_state(self, result) -> tuple: # this should be overridden pass def showReplyDetails(self, identifier: str, reqId: int): """ Accepts a request ID and prints the reply details :param identifier: Client's identifier :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(identifier, reqId) replyInfo = "Node {} replied with result {}" if replies: for frm, reply in replies.items(): print(replyInfo.format(frm, reply['result'])) else: print("No replies received from Nodes!") def onConnsChanged(self, joined: Set[str], left: Set[str]): """ Modify the current status of the client based on the status of the connections changed. """ if self.isGoing(): if len(self.nodestack.conns) == len(self.nodeReg): self.status = Status.started elif len(self.nodestack.conns) >= self.minNodesToConnect: self.status = Status.started_hungry self.flushMsgsPendingConnection() if self.ledger: for n in joined: self.sendLedgerStatus(n) @property def hasSufficientConnections(self): return len(self.nodestack.conns) >= self.minNodesToConnect @property def hasAnyConnections(self): return len(self.nodestack.conns) > 0 def can_send_write_requests(self): if not Mode.is_done_discovering(self.mode): return False if not self.hasSufficientConnections: return False return True def can_send_read_requests(self): if not Mode.is_done_discovering(self.mode): return False if not self.hasAnyConnections: return False return True def can_send_request(self, request): if not Mode.is_done_discovering(self.mode): return False if self.hasSufficientConnections: return True if not self.hasAnyConnections: return False if request.isForced(): return True is_read_only = request.txn_type in self._read_only_requests if is_read_only: return True return False def pendReqsTillConnection(self, request, signer=None): """ Enqueue requests that need to be submitted until the client has sufficient connections to nodes :return: """ self.reqsPendingConnection.append((request, signer)) logger.debug("{} enqueuing request since not enough connections " "with nodes: {}".format(self, request)) def flushMsgsPendingConnection(self): queueSize = len(self.reqsPendingConnection) if queueSize > 0: logger.debug("Flushing pending message queue of size {}" .format(queueSize)) tmp = deque() while self.reqsPendingConnection: req, signer = self.reqsPendingConnection.popleft() if self.can_send_request(req): self.send(req, signer=signer) else: tmp.append((req, signer)) self.reqsPendingConnection.extend(tmp) def _expect_replies(self, request: Request, nodes: Optional[Set[str]] = None): nodes = nodes if nodes else self._connected_node_names now = time.perf_counter() self.expectingAcksFor[request.key] = (nodes, now, 0) self.expectingRepliesFor[request.key] = (copy.copy(nodes), now, 0) self.startRepeating(self._retry_for_expected, self.config.CLIENT_REQACK_TIMEOUT) @property def _connected_node_names(self): return { remote.name for remote in self.nodestack.remotes.values() if self.nodestack.isRemoteConnected(remote) } def _got_expected(self, msg, sender): def drop(identifier, reqId, register): key = (identifier, reqId) if key in register: received = register[key][0] if sender in received: received.remove(sender) if not received: register.pop(key) if msg[OP_FIELD_NAME] == REQACK: drop(get_reply_itentifier(msg), get_reply_reqId(msg), self.expectingAcksFor) elif msg[OP_FIELD_NAME] == REPLY: drop(get_reply_itentifier(msg[f.RESULT.nm]), get_reply_reqId(msg[f.RESULT.nm]), self.expectingAcksFor) drop(get_reply_itentifier(msg[f.RESULT.nm]), get_reply_reqId(msg[f.RESULT.nm]), self.expectingRepliesFor) elif msg[OP_FIELD_NAME] in (REQNACK, REJECT): drop(get_reply_itentifier(msg), get_reply_reqId(msg), self.expectingAcksFor) drop(get_reply_itentifier(msg), get_reply_reqId(msg), self.expectingRepliesFor) else: raise RuntimeError("{} cannot retry {}".format(self, msg)) if not self.expectingAcksFor and not self.expectingRepliesFor: self._stop_expecting() def _stop_expecting(self): self.stopRepeating(self._retry_for_expected, strict=False) def _filter_expected(self, now, queue, retry_timeout, max_retry): dead_requests = [] alive_requests = {} not_answered_nodes = set() for requestKey, (expected_from, last_tried, retries) in queue.items(): if now < last_tried + retry_timeout: continue if retries >= max_retry: dead_requests.append(requestKey) continue if requestKey not in alive_requests: alive_requests[requestKey] = set() alive_requests[requestKey].update(expected_from) not_answered_nodes.update(expected_from) return dead_requests, alive_requests, not_answered_nodes def _retry_for_expected(self): now = time.perf_counter() requests_with_no_ack, alive_requests, not_acked_nodes = \ self._filter_expected(now, self.expectingAcksFor, self.config.CLIENT_REQACK_TIMEOUT, self.config.CLIENT_MAX_RETRY_ACK) requests_with_no_reply, alive_requests, not_replied_nodes = \ self._filter_expected(now, self.expectingRepliesFor, self.config.CLIENT_REPLY_TIMEOUT, self.config.CLIENT_MAX_RETRY_REPLY) for request_key in requests_with_no_ack: logger.debug('{} have got no ACKs for {} and will not try again' .format(self, request_key)) self.expectingAcksFor.pop(request_key) for request_key in requests_with_no_reply: logger.debug('{} have got no REPLYs for {} and will not try again' .format(self, request_key)) self.expectingRepliesFor.pop(request_key) if not_acked_nodes: logger.debug('{} going to retry for {}' .format(self, self.expectingAcksFor.keys())) for node_name in not_acked_nodes: try: remote = self.nodestack.getRemote(node_name) except RemoteNotFound: logger.warning('{}{} could not find remote {}' .format(CONNECTION_PREFIX, self, node_name)) continue logger.debug('Remote {} of {} being joined since REQACK for not ' 'received for request'.format(remote, self)) # This makes client to reconnect # even if pool is just busy and cannot answer quickly, # that's why using maintainConnections instead # self.nodestack.connect(name=remote.name) self.nodestack.maintainConnections(force=True) if alive_requests: # Need a delay in case connection has to be established with some # nodes, a better way is not to assume the delay value but only # send requests once the connection is established. Also it is # assumed that connection is not established if a node not sending # REQACK/REQNACK/REJECT/REPLY, but a little better way is to compare # the value in stats of the stack and look for changes in count of # `message_reject_rx` but that is not very helpful either since # it does not record which node rejected delay = 3 if not_acked_nodes else 0 self._schedule(partial(self.resendRequests, alive_requests), delay) def resendRequests(self, keys, force_expect=False): for key, nodes in keys.items(): if not nodes: continue request = self.reqRepStore.getRequest(*key) logger.debug('{} resending request {} to {}'. format(self, request, nodes)) self.sendToNodes(request, nodes) now = time.perf_counter() for queue in [self.expectingAcksFor, self.expectingRepliesFor]: if key in queue: _, _, retries = queue[key] queue[key] = (nodes, now, retries + 1) elif force_expect: queue[key] = (nodes, now, 1) def sendLedgerStatus(self, nodeName: str): ledgerStatus = LedgerStatus( POOL_LEDGER_ID, self.ledger.size, None, None, self.ledger.root_hash) rid = self.nodestack.getRemote(nodeName).uid self.send(ledgerStatus, rid) def send(self, msg: Any, *rids: Iterable[int], signer: Signer = None): return self.nodestack.send(msg, *rids, signer=signer) def sendToNodes(self, msg: Any, names: Iterable[str]): rids = [rid for rid, r in self.nodestack.remotes.items() if r.name in names] return self.send(msg, *rids) @staticmethod def verifyMerkleProof(*replies: Tuple[Reply]) -> bool: """ Verifies the correctness of the merkle proof provided in the reply from the node. Returns True if verified to be correct, throws an exception otherwise. :param replies: One or more replies for which Merkle Proofs have to be verified :raises ProofError: The proof is invalid :return: True """ verifier = MerkleVerifier() serializer = ledger_txn_serializer ignored = {F.auditPath.name, F.seqNo.name, F.rootHash.name} for r in replies: seqNo = r[f.RESULT.nm][F.seqNo.name] rootHash = Ledger.strToHash( r[f.RESULT.nm][F.rootHash.name]) auditPath = [Ledger.strToHash(a) for a in r[f.RESULT.nm][F.auditPath.name]] filtered = dict((k, v) for (k, v) in r[f.RESULT.nm].items() if k not in ignored) result = serializer.serialize(filtered) verifier.verify_leaf_inclusion(result, seqNo - 1, auditPath, STH(tree_size=seqNo, sha256_root_hash=rootHash)) return True def registerObserver(self, observer: Callable, name=None): if not name: name = uuid.uuid4() if name in self._observers or observer in self._observerSet: raise RuntimeError("Observer {} already registered".format(name)) self._observers[name] = observer self._observerSet.add(observer) def deregisterObserver(self, name): if name not in self._observers: raise RuntimeError("Observer {} not registered".format(name)) self._observerSet.remove(self._observers[name]) del self._observers[name] def hasObserver(self, observer): return observer in self._observerSet
class Client(Motor, MessageProcessor, HasFileStorage, HasPoolManager, HasActionQueue): def __init__(self, name: str, nodeReg: Dict[str, HA]=None, ha: Union[HA, Tuple[str, int]]=None, basedirpath: str=None, config=None, sighex: str=None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName or 'Client~' + str(id(self)) cha = None # If client information already exists is RAET then use that if self.exists(self.stackName, basedirpath): cha = self.nodeStackClass.getHaFromLocal(self.stackName, basedirpath) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) # TODO: Find a proper name self.alias = name self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger(POOL_LEDGER_ID, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setF() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auth_mode=AuthMode.ALLOW_ANY.value) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable # TODO I think this is a bug here, sighex is getting passed in the seed parameter self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info("Client {} initialized with the following node registry:" .format(self.alias)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp)) def getReqRepStore(self): return ClientReqRepStoreFile(self.name, self.basedirpath) def getTxnLogStore(self): return ClientTxnLog(self.name, self.basedirpath) def __repr__(self): return self.name def postPoolLedgerCaughtUp(self): self.mode = Mode.discovered # For the scenario where client has already connected to nodes reading # the genesis pool transactions and that is enough self.flushMsgsPendingConnection() def postTxnFromCatchupAddedToLedger(self, ledgerType: int, txn: Any): if ledgerType != 0: logger.error("{} got unknown ledger type {}". format(self, ledgerType)) return self.processPoolTxn(txn) # noinspection PyAttributeOutsideInit def setF(self): nodeCount = len(self.nodeReg) self.f = getMaxFailures(nodeCount) self.minNodesToConnect = self.f + 1 self.totalNodes = nodeCount self.quorums = Quorums(nodeCount) @staticmethod def exists(name, basedirpath): return os.path.exists(basedirpath) and \ os.path.exists(os.path.join(basedirpath, name)) @property def nodeStackClass(self) -> NetworkInterface: return nodeStackClass def start(self, loop): oldstatus = self.status if oldstatus in Status.going(): logger.info("{} is already {}, so start has no effect". format(self.alias, self.status.name)) else: super().start(loop) self.nodestack.start() self.nodestack.maintainConnections(force=True) if self._ledger: self.ledgerManager.setLedgerCanSync(0, True) self.mode = Mode.starting async def prod(self, limit) -> int: """ async function that returns the number of events :param limit: The number of messages to be processed :return: The number of events up to a prescribed `limit` """ s = 0 if self.isGoing(): s = await self.nodestack.service(limit) self.nodestack.serviceLifecycle() self.nodestack.flushOutBoxes() s += self._serviceActions() # TODO: This if condition has to be removed. `_ledger` if once set wont # be reset ever so in `__init__` the `prod` method should be patched. if self._ledger: s += self.ledgerManager._serviceActions() return s def submitReqs(self, *reqs: Request) -> List[Request]: requests = [] for request in reqs: if (self.mode == Mode.discovered and self.hasSufficientConnections) or \ (request.isForced() and self.hasAnyConnections): logger.debug('Client {} sending request {}'.format(self, request)) self.send(request) self.expectingFor(request) else: logger.debug("{} pending request since in mode {} and " "connected to {} nodes". format(self, self.mode, self.nodestack.connecteds)) self.pendReqsTillConnection(request) requests.append(request) for r in requests: self.reqRepStore.addRequest(r) return requests def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None) -> None: """ Handles single message from a node, and appends it to a queue :param wrappedMsg: Reply received by the client from the node """ self.inBox.append(wrappedMsg) msg, frm = wrappedMsg # Do not print result of transaction type `POOL_LEDGER_TXNS` on the CLI ledgerTxnTypes = (POOL_LEDGER_TXNS, LEDGER_STATUS, CONSISTENCY_PROOF, CATCHUP_REP) printOnCli = not excludeFromCli and msg.get(OP_FIELD_NAME) not \ in ledgerTxnTypes logger.info("Client {} got msg from node {}: {}". format(self.name, frm, msg), extra={"cli": printOnCli}) if OP_FIELD_NAME in msg: if msg[OP_FIELD_NAME] in ledgerTxnTypes and self._ledger: cMsg = node_message_factory.get_instance(**msg) if msg[OP_FIELD_NAME] == POOL_LEDGER_TXNS: self.poolTxnReceived(cMsg, frm) if msg[OP_FIELD_NAME] == LEDGER_STATUS: self.ledgerManager.processLedgerStatus(cMsg, frm) if msg[OP_FIELD_NAME] == CONSISTENCY_PROOF: self.ledgerManager.processConsistencyProof(cMsg, frm) if msg[OP_FIELD_NAME] == CATCHUP_REP: self.ledgerManager.processCatchupRep(cMsg, frm) elif msg[OP_FIELD_NAME] == REQACK: self.reqRepStore.addAck(msg, frm) self.gotExpected(msg, frm) elif msg[OP_FIELD_NAME] == REQNACK: self.reqRepStore.addNack(msg, frm) self.gotExpected(msg, frm) elif msg[OP_FIELD_NAME] == REJECT: self.reqRepStore.addReject(msg, frm) self.gotExpected(msg, frm) elif msg[OP_FIELD_NAME] == REPLY: result = msg[f.RESULT.nm] identifier = msg[f.RESULT.nm][f.IDENTIFIER.nm] reqId = msg[f.RESULT.nm][f.REQ_ID.nm] numReplies = self.reqRepStore.addReply(identifier, reqId, frm, result) self.gotExpected(msg, frm) self.postReplyRecvd(identifier, reqId, frm, result, numReplies) def postReplyRecvd(self, identifier, reqId, frm, result, numReplies): if not self.txnLog.hasTxn(identifier, reqId) and numReplies > self.f: replies = self.reqRepStore.getReplies(identifier, reqId).values() reply = checkIfMoreThanFSameItems(replies, self.f) if reply: self.txnLog.append(identifier, reqId, reply) return reply def _statusChanged(self, old, new): # do nothing for now pass def onStopping(self, *args, **kwargs): logger.debug('Stopping client {}'.format(self)) self.nodestack.nextCheck = 0 self.nodestack.stop() if self._ledger: self.ledgerManager.setLedgerState(POOL_LEDGER_ID, LedgerState.not_synced) self.mode = None self._ledger.stop() if self.hashStore and not self.hashStore.closed: self.hashStore.close() self.txnLog.close() def getReply(self, identifier: str, reqId: int) -> Optional[Reply]: """ Accepts reply message from node if the reply is matching :param identifier: identifier of the entity making the request :param reqId: Request Id :return: Reply message only when valid and matching (None, NOT_FOUND) (None, UNCONFIRMED) f+1 not reached (reply, CONFIRMED) f+1 reached """ try: cons = self.hasConsensus(identifier, reqId) except KeyError: return None, "NOT_FOUND" if cons: return cons, "CONFIRMED" return None, "UNCONFIRMED" def getRepliesFromAllNodes(self, identifier: str, reqId: int): """ Accepts a request ID and return a list of results from all the nodes for that request :param identifier: identifier of the entity making the request :param reqId: Request ID :return: list of request results from all nodes """ return {frm: msg for msg, frm in self.inBox if msg[OP_FIELD_NAME] == REPLY and msg[f.RESULT.nm][f.REQ_ID.nm] == reqId and msg[f.RESULT.nm][f.IDENTIFIER.nm] == identifier} def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]: """ Accepts a request ID and returns True if consensus was reached for the request or else False :param identifier: identifier of the entity making the request :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(identifier, reqId) if not replies: raise KeyError('{}{}'.format(identifier, reqId)) # NOT_FOUND # Check if at least f+1 replies are received or not. if self.quorums.reply.is_reached(len(replies)): onlyResults = {frm: reply["result"] for frm, reply in replies.items()} resultsList = list(onlyResults.values()) # if all the elements in the resultList are equal - consensus # is reached. if all(result == resultsList[0] for result in resultsList): return resultsList[0] # CONFIRMED else: logger.error( "Received a different result from at least one of the nodes..") return checkIfMoreThanFSameItems(resultsList, self.f) else: return False # UNCONFIRMED def showReplyDetails(self, identifier: str, reqId: int): """ Accepts a request ID and prints the reply details :param identifier: Client's identifier :param reqId: Request ID """ replies = self.getRepliesFromAllNodes(identifier, reqId) replyInfo = "Node {} replied with result {}" if replies: for frm, reply in replies.items(): print(replyInfo.format(frm, reply['result'])) else: print("No replies received from Nodes!") def onConnsChanged(self, joined: Set[str], left: Set[str]): """ Modify the current status of the client based on the status of the connections changed. """ if self.isGoing(): if len(self.nodestack.conns) == len(self.nodeReg): self.status = Status.started elif len(self.nodestack.conns) >= self.minNodesToConnect: self.status = Status.started_hungry self.flushMsgsPendingConnection() if self._ledger: for n in joined: self.sendLedgerStatus(n) def replyIfConsensus(self, identifier, reqId: int): replies, errors = self.reqRepStore.getAllReplies(identifier, reqId) r = list(replies.values())[0] if len(replies) > self.f else None e = list(errors.values())[0] if len(errors) > self.f else None return r, e @property def hasSufficientConnections(self): return len(self.nodestack.conns) >= self.minNodesToConnect @property def hasAnyConnections(self): return len(self.nodestack.conns) > 0 def hasMadeRequest(self, identifier, reqId: int): return self.reqRepStore.hasRequest(identifier, reqId) def isRequestSuccessful(self, identifier, reqId): acks = self.reqRepStore.getAcks(identifier, reqId) nacks = self.reqRepStore.getNacks(identifier, reqId) f = getMaxFailures(len(self.nodeReg)) if len(acks) > f: return True, "Done" elif len(nacks) > f: # TODO: What if the the nacks were different from each node? return False, list(nacks.values())[0] else: return None def pendReqsTillConnection(self, request, signer=None): """ Enqueue requests that need to be submitted until the client has sufficient connections to nodes :return: """ self.reqsPendingConnection.append((request, signer)) logger.debug("{} enqueuing request since not enough connections " "with nodes: {}".format(self, request)) def flushMsgsPendingConnection(self): queueSize = len(self.reqsPendingConnection) if queueSize > 0: logger.debug("Flushing pending message queue of size {}" .format(queueSize)) tmp = deque() while self.reqsPendingConnection: req, signer = self.reqsPendingConnection.popleft() if (self.hasSufficientConnections and self.mode == Mode.discovered) or \ (req.isForced() and self.hasAnyConnections): self.send(req, signer=signer) else: tmp.append((req, signer)) self.reqsPendingConnection.extend(tmp) def expectingFor(self, request: Request, nodes: Optional[Set[str]]=None): nodes = nodes or {r.name for r in self.nodestack.remotes.values() if self.nodestack.isRemoteConnected(r)} now = time.perf_counter() self.expectingAcksFor[request.key] = (nodes, now, 0) self.expectingRepliesFor[request.key] = (copy.copy(nodes), now, 0) self.startRepeating(self.retryForExpected, self.config.CLIENT_REQACK_TIMEOUT) def gotExpected(self, msg, frm): if msg[OP_FIELD_NAME] == REQACK: container = msg colls = (self.expectingAcksFor, ) elif msg[OP_FIELD_NAME] == REPLY: container = msg[f.RESULT.nm] # If an REQACK sent by node was lost, the request when sent again # would fetch the reply or the client might just lose REQACK and not # REPLY so when REPLY received, request does not need to be resent colls = (self.expectingAcksFor, self.expectingRepliesFor) elif msg[OP_FIELD_NAME] in (REQNACK, REJECT): container = msg colls = (self.expectingAcksFor, self.expectingRepliesFor) else: raise RuntimeError("{} cannot retry {}".format(self, msg)) idr = container.get(f.IDENTIFIER.nm) reqId = container.get(f.REQ_ID.nm) key = (idr, reqId) for coll in colls: if key in coll: if frm in coll[key][0]: coll[key][0].remove(frm) if not coll[key][0]: coll.pop(key) if not (self.expectingAcksFor or self.expectingRepliesFor): self.stopRetrying() def stopRetrying(self): self.stopRepeating(self.retryForExpected, strict=False) def _filterExpected(self, now, queue, retryTimeout, maxRetry): deadRequests = [] aliveRequests = {} notAnsweredNodes = set() for requestKey, (expectedFrom, lastTried, retries) in queue.items(): if now < lastTried + retryTimeout: continue if retries >= maxRetry: deadRequests.append(requestKey) continue if requestKey not in aliveRequests: aliveRequests[requestKey] = set() aliveRequests[requestKey].update(expectedFrom) notAnsweredNodes.update(expectedFrom) return deadRequests, aliveRequests, notAnsweredNodes def retryForExpected(self): now = time.perf_counter() requestsWithNoAck, aliveRequests, notAckedNodes = \ self._filterExpected(now, self.expectingAcksFor, self.config.CLIENT_REQACK_TIMEOUT, self.config.CLIENT_MAX_RETRY_ACK) requestsWithNoReply, aliveRequests, notRepliedNodes = \ self._filterExpected(now, self.expectingRepliesFor, self.config.CLIENT_REPLY_TIMEOUT, self.config.CLIENT_MAX_RETRY_REPLY) for requestKey in requestsWithNoAck: logger.debug('{} have got no ACKs for {} and will not try again' .format(self, requestKey)) self.expectingAcksFor.pop(requestKey) for requestKey in requestsWithNoReply: logger.debug('{} have got no REPLYs for {} and will not try again' .format(self, requestKey)) self.expectingRepliesFor.pop(requestKey) if notAckedNodes: logger.debug('{} going to retry for {}' .format(self, self.expectingAcksFor.keys())) for nm in notAckedNodes: try: remote = self.nodestack.getRemote(nm) except RemoteNotFound: logger.warning('{} could not find remote {}'.format(self, nm)) continue logger.debug('Remote {} of {} being joined since REQACK for not ' 'received for request'.format(remote, self)) # This makes client to reconnect # even if pool is just busy and cannot answer quickly, # that's why using maintainConnections instead # self.nodestack.connect(name=remote.name) self.nodestack.maintainConnections(force=True) if aliveRequests: # Need a delay in case connection has to be established with some # nodes, a better way is not to assume the delay value but only # send requests once the connection is established. Also it is # assumed that connection is not established if a node not sending # REQACK/REQNACK/REJECT/REPLY, but a little better way is to compare # the value in stats of the stack and look for changes in count of # `message_reject_rx` but that is not very helpful either since # it does not record which node rejected delay = 3 if notAckedNodes else 0 self._schedule(partial(self.resendRequests, aliveRequests), delay) def resendRequests(self, keys): for key, nodes in keys.items(): if not nodes: continue request = self.reqRepStore.getRequest(*key) logger.debug('{} resending request {} to {}'. format(self, request, nodes)) self.sendToNodes(request, nodes) now = time.perf_counter() for queue in [self.expectingAcksFor, self.expectingRepliesFor]: if key in queue: _, _, retries = queue[key] queue[key] = (nodes, now, retries + 1) def sendLedgerStatus(self, nodeName: str): ledgerStatus = LedgerStatus(POOL_LEDGER_ID, self.ledger.size, None, None, self.ledger.root_hash) rid = self.nodestack.getRemote(nodeName).uid self.send(ledgerStatus, rid) def send(self, msg: Any, *rids: Iterable[int], signer: Signer = None): self.nodestack.send(msg, *rids, signer=signer) def sendToNodes(self, msg: Any, names: Iterable[str]): rids = [rid for rid, r in self.nodestack.remotes.items() if r.name in names] self.send(msg, *rids) @staticmethod def verifyMerkleProof(*replies: Tuple[Reply]) -> bool: """ Verifies the correctness of the merkle proof provided in the reply from the node. Returns True if verified to be correct, throws an exception otherwise. :param replies: One or more replies for which Merkle Proofs have to be verified :raises ProofError: The proof is invalid :return: True """ verifier = MerkleVerifier() fields = getTxnOrderedFields() serializer = CompactSerializer(fields=fields) ignored = {F.auditPath.name, F.seqNo.name, F.rootHash.name, TXN_TIME} for r in replies: seqNo = r[f.RESULT.nm][F.seqNo.name] rootHash = Ledger.strToHash( r[f.RESULT.nm][F.rootHash.name]) auditPath = [Ledger.strToHash(a) for a in r[f.RESULT.nm][F.auditPath.name]] filtered = dict((k, v) for (k, v) in r[f.RESULT.nm].items() if k not in ignored) result = serializer.serialize(filtered) verifier.verify_leaf_inclusion(result, seqNo - 1, auditPath, STH(tree_size=seqNo, sha256_root_hash=rootHash)) return True
def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, genesis_dir: str = None, ledger_dir: str = None, keys_dir: str = None, plugins_dir: str = None, config=None, sighex: str = None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() dataDir = self.config.clientDataDir or "data/clients" self.basedirpath = basedirpath or self.config.CLI_BASE_DIR self.basedirpath = os.path.expanduser(self.basedirpath) signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName or 'Client~' + str(id(self)) self.genesis_dir = genesis_dir or self.basedirpath self.ledger_dir = ledger_dir or os.path.join(self.basedirpath, dataDir, self.name) self.plugins_dir = plugins_dir or self.basedirpath _keys_dir = keys_dir or self.basedirpath self.keys_dir = os.path.join(_keys_dir, "keys") cha = None if self.exists(self.stackName, self.keys_dir): cha = self.nodeStackClass.getHaFromLocal( self.stackName, self.keys_dir) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() HasFileStorage.__init__(self, self.ledger_dir) # TODO: Find a proper name self.alias = name if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( POOL_LEDGER_ID, self.ledger, preCatchupStartClbk=self.prePoolLedgerCatchup, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setPoolParams() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auth_mode=AuthMode.ALLOW_ANY.value) stackargs['basedirpath'] = self.keys_dir self.created = time.perf_counter() # noinspection PyCallingNonCallable # TODO I think this is a bug here, sighex is getting passed in the seed # parameter self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info( "Client {} initialized with the following node registry:".format( self.alias)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} self._observers = {} # type Dict[str, Callable] self._observerSet = set() # makes it easier to guard against duplicates plugins_to_load = self.config.PluginsToLoad if hasattr(self.config, "PluginsToLoad") else None tp = loadPlugins(self.plugins_dir, plugins_to_load) logger.debug("total plugins loaded in client: {}".format(tp)) self._multi_sig_verifier = self._create_multi_sig_verifier() self._read_only_requests = set()
def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, config=None, sighex: str = None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName cha = None # If client information already exists is RAET then use that if self.exists(self.stackName, basedirpath): cha = getHaFromLocalEstate(self.stackName, basedirpath) if cha: cha = HA(*cha) logger.debug( "Client {} ignoring given ha {} and using {}".format( self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) # TODO: Find a proper name self.alias = name self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( 0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setF() stackargs = dict( name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auto=2) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info( "Client {} initialized with the following node registry:". format(self.alias)) lengths = [ max(x) for x in zip(*[(len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()]) ] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info("Client {} found an empty node registry:".format( self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp))