def orderer(consensus_data, internal_bus, external_bus, name, write_manager, txn_roots, state_roots, bls_bft_replica, tconf, stasher): orderer = OrderingService( data=consensus_data(name), timer=QueueTimer(), bus=internal_bus, network=external_bus, write_manager=write_manager, bls_bft_replica=bls_bft_replica, freshness_checker=FreshnessChecker( freshness_timeout=tconf.STATE_FRESHNESS_UPDATE_INTERVAL), stasher=stasher) orderer._data.node_mode = Mode.participating orderer._data.primary_name = "Alpha:0" orderer.get_txn_root_hash = lambda ledger, to_str=False: txn_roots[ledger] orderer.get_state_root_hash = lambda ledger, to_str=False: state_roots[ ledger] orderer.requestQueues[DOMAIN_LEDGER_ID] = OrderedSet() orderer._revert = lambda *args, **kwargs: None orderer.db_manager.stores[LAST_SENT_PP_STORE_LABEL] = \ FakeSomething(store_last_sent_pp_seq_no=lambda b, c: None) future_primaries_handler = FuturePrimariesBatchHandler( write_manager.database_manager, FakeSomething(nodeReg={}, nodeIds=[])) future_primaries_handler.get_primaries = lambda *args, **kwargs: orderer._data.primaries write_manager.register_batch_handler(future_primaries_handler) return orderer
def fake_view_changer(request, tconf): node_count = 4 node_stack = FakeSomething( name="fake stack", connecteds={"Alpha", "Beta", "Gamma", "Delta"}, conns={"Alpha", "Beta", "Gamma", "Delta"} ) monitor = FakeSomething( isMasterDegraded=lambda: False, areBackupsDegraded=lambda: [], prettymetrics='' ) node = FakeSomething( name="SomeNode", timer=QueueTimer(), viewNo=request.param, quorums=Quorums(getValueFromModule(request, 'nodeCount', default=node_count)), nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), config=tconf, monitor=monitor, discard=lambda a, b, c, d: print(b), primaries_disconnection_times=[None] * getRequiredInstances(node_count), master_primary_name='Alpha', master_replica=FakeSomething(instId=0), nodeStatusDB=None ) view_changer = create_view_changer(node) # TODO: This is a hack for tests compatibility, do something better view_changer.node = node return view_changer
def fake_node(tconf): node = FakeSomething( config=tconf, timer=QueueTimer(), nodeStatusDB=None, master_replica=FakeSomething( inBox=deque(), inBoxRouter=Router(), logger=FakeSomething(info=lambda *args, **kwargs: True)), name="Alpha", master_primary_name="Alpha", on_view_change_start=lambda *args, **kwargs: True, start_catchup=lambda *args, **kwargs: True, nodeInBox=deque(), nodeMsgRouter=Router(), metrics=None, process_one_node_message=None, quota_control=FakeSomething(node_quota=Quota(count=100, size=100)), nodestack=FakeSomething( service=lambda *args, **kwargs: eventually(lambda: True)), set_view_for_replicas=lambda view_no: None, set_view_change_status=lambda view_no: None) node.metrics = functools.partial(Node._createMetricsCollector, node)() node.process_one_node_message = functools.partial( Node.process_one_node_message, node) return node
def __init__(self, viewNo, quorums, ledger_ids): node_names = ["Alpha", "Beta", "Gamma", "Delta"] node_stack = FakeSomething( name="fake stack", connecteds=set(node_names) ) self.replicas = [] self.viewNo = viewNo audit_ledger = FakeSomething(size=0) db_manager = DatabaseManager() db_manager.register_new_database(AUDIT_LEDGER_ID, audit_ledger) super().__init__( name="fake node", ledger_ids=ledger_ids, _viewNo=viewNo, quorums=quorums, nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), mode=Mode.participating, view_change_in_progress=False, requests=Requests(), onBatchCreated=lambda self, *args, **kwargs: True, applyReq=lambda self, *args, **kwargs: True, primaries=[], get_validators=lambda: [], db_manager=db_manager, write_manager=FakeSomething(database_manager=db_manager, apply_request=lambda req, cons_time: None), timer=QueueTimer(), poolManager=FakeSomething(node_names_ordered_by_rank=lambda: node_names) )
def __init__(self, viewNo, quorums, ledger_ids): node_stack = FakeSomething( name="fake stack", connecteds={"Alpha", "Beta", "Gamma", "Delta"}) self.replicas = [] self.viewNo = viewNo super().__init__(name="fake node", ledger_ids=ledger_ids, _viewNo=viewNo, quorums=quorums, nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), mode=Mode.participating, view_change_in_progress=False, pre_view_change_in_progress=False, requests=Requests(), onBatchCreated=lambda self, *args, **kwargs: True, applyReq=lambda self, *args, **kwargs: True, primaries=[], get_validators=lambda: [], db_manager=None, internal_bus=InternalBus(), write_manager=FakeSomething( database_manager=DatabaseManager(), apply_request=lambda req, cons_time: None), timer=QueueTimer())
def orderer(consensus_data, internal_bus, external_bus, name, write_manager, txn_roots, state_roots, bls_bft_replica, tconf, stasher, validators): orderer = OrderingService( data=consensus_data(name), timer=QueueTimer(), bus=internal_bus, network=external_bus, write_manager=write_manager, bls_bft_replica=bls_bft_replica, freshness_checker=FreshnessChecker( freshness_timeout=tconf.STATE_FRESHNESS_UPDATE_INTERVAL), primaries_selector=RoundRobinConstantNodesPrimariesSelector( validators), stasher=stasher) orderer._data.node_mode = Mode.participating orderer._data.primary_name = "Alpha:0" orderer.get_txn_root_hash = lambda ledger, to_str=False: txn_roots[ledger] orderer.get_state_root_hash = lambda ledger, to_str=False: state_roots[ ledger] orderer.requestQueues[DOMAIN_LEDGER_ID] = OrderedSet() orderer._revert = lambda *args, **kwargs: None orderer.db_manager.stores[LAST_SENT_PP_STORE_LABEL] = \ FakeSomething(store_last_sent_pp_seq_no=lambda b, c: None) return orderer
def test_timer_cancel_callback_doesnt_crash_for_nonexistant_callback(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb = Callback() # This shouldn't crash timer.cancel(cb) # Make sure that callback which was scheduled later is still called timer.schedule(5, cb) ts.value += 6 timer.service() assert cb.call_count == 1 # And this still shouldn't crash timer.cancel(cb)
def test_timer_can_cancel_callback(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb = Callback() timer.schedule(5, cb) ts.value += 3 timer.service() assert cb.call_count == 0 timer.cancel(cb) ts.value += 3 timer.service() assert cb.call_count == 0
def test_timer_can_cancel_callback_without_touching_other_callbacks(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb1 = Callback() cb2 = Callback() cb3 = Callback() timer.schedule(5, cb1) timer.schedule(3, cb2) timer.schedule(4, cb3) timer.cancel(cb2) ts.value += 6 timer.service() assert cb1.call_count == 1 assert cb2.call_count == 0 assert cb3.call_count == 1
def __init__(self, name, config, prepare_to_send, metrics, mt_outgoing_size, timer=None, listener=None): self._name = name self.metrics = metrics self.listener = listener self._prepare_to_send = prepare_to_send self._mt_outgoing_size = mt_outgoing_size self._config = config self._timer = QueueTimer() if timer is None else timer self._pending_client_messages = OrderedDict() RepeatingTimer(self._timer, self._config.RESEND_CLIENT_MSG_TIMEOUT, self._send_pending_messages) RepeatingTimer(self._timer, self._config.REMOVE_CLIENT_MSG_TIMEOUT, self._remove_old_messages)
def test_timer_can_schedule_callback_twice(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb = Callback() timer.schedule(3, cb) timer.schedule(5, cb) ts.value += 4 timer.service() assert cb.call_count == 1 ts.value += 4 timer.service() assert cb.call_count == 2
def __init__(self, tmpdir, config=None): node_names = ['Node1', 'Node2', 'Node3', 'Node4'] self.basedirpath = tmpdir self.name = node_names[0] self.viewNo = 0 self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = node_names self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.poolManager = FakeSomething( node_names_ordered_by_rank=lambda: node_names) self.mode = Mode.starting self.monitor = FakeSomething(isMasterDegraded=lambda: False) self.config = config or getConfigOnce() self.nodeStatusDB = None self.quorums = Quorums(self.totalNodes) self.nodestack = FakeSomething(connecteds=set(self.allNodeNames)) self.write_manager = FakeSomething( node_reg_handler=NodeRegHandler(self.db_manager)) self.primaries_selector = RoundRobinConstantNodesPrimariesSelector( node_names) self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config) } self.requiredNumberOfInstances = 2 self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.metrics = NullMetricsCollector() # For catchup testing self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def orderer(consensus_data, internal_bus, external_bus, name, write_manager, txn_roots, state_roots, bls_bft_replica): orderer = OrderingService(data=consensus_data(name), timer=QueueTimer(), bus=internal_bus, network=external_bus, write_manager=write_manager, bls_bft_replica=bls_bft_replica, is_master=is_master) orderer._data.is_participating = True orderer.primary_name = "Alpha:0" orderer.l_txnRootHash = lambda ledger, to_str=False: txn_roots[ledger] orderer.l_stateRootHash = lambda ledger, to_str=False: state_roots[ledger] orderer.requestQueues[DOMAIN_LEDGER_ID] = OrderedSet() orderer.l_revert = lambda *args, **kwargs: None return orderer
def test_timer_can_schedule_different_callbacks(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb1 = Callback() cb2 = Callback() timer.schedule(5, cb1) timer.schedule(3, cb2) ts.value += 4 timer.service() assert cb1.call_count == 0 assert cb2.call_count == 1 ts.value += 4 timer.service() assert cb1.call_count == 1 assert cb2.call_count == 1
def test_timer_triggers_callback_on_time(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb = Callback() timer.schedule(5, cb) assert cb.call_count == 0 ts.value += 5 timer.service() assert cb.call_count == 1
def test_primary_names_cleaning(tconf): node = FakeSomething( name="fake node", ledger_ids=[0], viewNo=0, utc_epoch=get_utc_epoch, get_validators=lambda: [], db_manager=DatabaseManager(), requests=[], mode=Mode.participating, timer=QueueTimer(), quorums=Quorums(4), write_manager=None, poolManager=FakeSomething(node_names_ordered_by_rank=lambda: []), primaries_selector=RoundRobinConstantNodesPrimariesSelector( ["Alpha", "Beta", "Gamma", "Delta"])) bls_bft_replica = FakeSomething(gc=lambda *args: None, ) replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica) replica.primaryName = "Node1:0" assert list(replica.primaryNames.items()) == \ [(0, "Node1:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node2:0" assert list(replica.primaryNames.items()) == \ [(0, "Node1:0"), (1, "Node2:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node3:0" assert list(replica.primaryNames.items()) == \ [(1, "Node2:0"), (2, "Node3:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node4:0" assert list(replica.primaryNames.items()) == \ [(2, "Node3:0"), (3, "Node4:0")]
def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.internal_bus = InternalBus() self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.nodeStatusDB = None self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = create_view_changer(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def test_timer_can_schedule_and_simultaneously_process_different_callbacks(): ts = MockTimestamp(0) timer = QueueTimer(ts) cb1 = Callback() cb2 = Callback() timer.schedule(5, cb1) timer.schedule(3, cb2) ts.value += 6 timer.service() assert cb1.call_count == 1 assert cb2.call_count == 1
def test_ordered_cleaning(tconf): global_view_no = 2 node = FakeSomething( name="fake node", ledger_ids=[0], viewNo=global_view_no, utc_epoch=get_utc_epoch, get_validators=lambda: [], db_manager=DatabaseManager(), requests=[], mode=Mode.participating, timer=QueueTimer(), quorums=Quorums(4), write_manager=None, poolManager=FakeSomething(node_names_ordered_by_rank=lambda: []), primaries_selector=RoundRobinConstantNodesPrimariesSelector( ["Alpha", "Beta", "Gamma", "Delta"])) bls_bft_replica = FakeSomething(gc=lambda *args: None, ) replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica) replica._consensus_data.view_no = global_view_no total = [] num_requests_per_view = 3 for viewNo in range(global_view_no + 1): for seqNo in range(num_requests_per_view): reqId = viewNo, seqNo replica._ordering_service._add_to_ordered(*reqId) total.append(reqId) # gc is called after stable checkpoint, since no request executed # in this test starting it manually replica._ordering_service.gc(100) # Requests with view lower then previous view # should not be in ordered assert len(replica._ordering_service.ordered) == len( total[num_requests_per_view:])
def test_primary_names_cleaning(tconf): node = FakeSomething(name="fake node", ledger_ids=[0], viewNo=0, utc_epoch=get_utc_epoch, get_validators=lambda: [], internal_bus=InternalBus(), db_manager=DatabaseManager(), requests=[], mode=Mode.participating, timer=QueueTimer(), quorums=Quorums(4), write_manager=None) bls_bft_replica = FakeSomething(gc=lambda *args: None, ) replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica) replica.primaryName = "Node1:0" assert list(replica.primaryNames.items()) == \ [(0, "Node1:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node2:0" assert list(replica.primaryNames.items()) == \ [(0, "Node1:0"), (1, "Node2:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node3:0" assert list(replica.primaryNames.items()) == \ [(1, "Node2:0"), (2, "Node3:0")] node.viewNo += 1 replica._consensus_data.view_no = node.viewNo replica.primaryName = "Node4:0" assert list(replica.primaryNames.items()) == \ [(2, "Node3:0"), (3, "Node4:0")]
def __init__(self, get_current_time: Optional[MockTimestamp] = None): self._ts = get_current_time if get_current_time else MockTimestamp(0) QueueTimer.__init__(self, self._ts)
class ClientMessageProvider: def __init__(self, name, config, prepare_to_send, metrics, mt_outgoing_size, timer=None, listener=None): self._name = name self.metrics = metrics self.listener = listener self._prepare_to_send = prepare_to_send self._mt_outgoing_size = mt_outgoing_size self._config = config self._timer = QueueTimer() if timer is None else timer self._pending_client_messages = OrderedDict() RepeatingTimer(self._timer, self._config.RESEND_CLIENT_MSG_TIMEOUT, self._send_pending_messages) RepeatingTimer(self._timer, self._config.REMOVE_CLIENT_MSG_TIMEOUT, self._remove_old_messages) def transmit_through_listener(self, msg, ident) -> Tuple[bool, Optional[str]]: self._pending_client_messages.setdefault(ident, []).append( (self._timer.get_current_time(), msg)) if len(self._pending_client_messages ) > self._config.PENDING_CLIENT_LIMIT: self._pending_client_messages.popitem(last=False) if len(self._pending_client_messages[ident] ) > self._config.PENDING_MESSAGES_FOR_ONE_CLIENT_LIMIT: self._pending_client_messages[ident].pop(0) result = True error_msg = None for timestamp, current_msg in list( self._pending_client_messages[ident]): result, error_msg, need_to_resend = self._transmit_one_msg_throughlistener( current_msg, ident) if not need_to_resend: self._remove_message(ident, (timestamp, current_msg)) return result, error_msg def _send_pending_messages(self): result = True error_msg = None for ident in list(self._pending_client_messages.keys()): for timestamp, current_msg in list( self._pending_client_messages[ident]): if self._timer.get_current_time( ) - timestamp >= self._config.RESEND_CLIENT_MSG_TIMEOUT: result, error_msg, need_to_resend = self._transmit_one_msg_throughlistener( current_msg, ident) if not need_to_resend: self._remove_message(ident, (timestamp, current_msg)) return result, error_msg def _transmit_one_msg_throughlistener( self, msg, ident) -> Tuple[bool, Optional[str], bool]: def prepare_error_msg(ex): err_str = '{}{} got error {} while sending through listener to {}' \ .format(CONNECTION_PREFIX, self, ex, ident) logger.warning(err_str) return err_str need_to_resend = False if isinstance(ident, str): ident = ident.encode() try: msg = self._prepare_to_send(msg) logger.trace( '{} transmitting {} to {} through listener socket'.format( self, msg, ident)) self.metrics.add_event(self._mt_outgoing_size, len(msg)) self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format( CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str, need_to_resend except zmq.Again as ex: need_to_resend = True return False, prepare_error_msg(ex), need_to_resend except zmq.ZMQError as ex: need_to_resend = (ex.errno == 113) return False, prepare_error_msg(ex), need_to_resend except Exception as ex: return False, prepare_error_msg(ex), need_to_resend return True, None, need_to_resend def _remove_old_messages(self): for ident in list(self._pending_client_messages.keys()): for timestamp, current_msg in list( self._pending_client_messages[ident]): if self._timer.get_current_time( ) - timestamp >= self._config.REMOVE_CLIENT_MSG_TIMEOUT: self._remove_message(ident, (timestamp, current_msg)) def _remove_message(self, ident, msg): self._pending_client_messages[ident].remove(msg) if not self._pending_client_messages[ident]: self._pending_client_messages.pop(ident) def __repr__(self): return self._name