def __init__(self, name: str, validators: List[str], primary_name: str, timer: TimerService, bus: InternalBus, network: ExternalBus, write_manager: WriteRequestManager, bls_bft_replica: BlsBftReplica = None): # ToDo: Maybe ConsensusSharedData should be initiated before and passed already prepared? self._internal_bus = bus self._data = ConsensusSharedData(name, validators, 0) self._data.primary_name = generateName(primary_name, self._data.inst_id) self.config = getConfig() self.stasher = StashingRouter(self.config.REPLICA_STASH_LIMIT, buses=[bus, network]) self._write_manager = write_manager self._primaries_selector = RoundRobinNodeRegPrimariesSelector( self._write_manager.node_reg_handler) self._orderer = OrderingService( data=self._data, timer=timer, bus=bus, network=network, write_manager=self._write_manager, bls_bft_replica=bls_bft_replica, freshness_checker=FreshnessChecker( freshness_timeout=self.config.STATE_FRESHNESS_UPDATE_INTERVAL), primaries_selector=self._primaries_selector, stasher=self.stasher) self._checkpointer = CheckpointService(self._data, bus, network, self.stasher, write_manager.database_manager) self._view_changer = ViewChangeService(self._data, timer, bus, network, self.stasher, self._primaries_selector) self._message_requestor = MessageReqService(self._data, bus, network) self._add_ledgers() # TODO: This is just for testing purposes only self._data.checkpoints.append( Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw')) # ToDo: it should be done in Zero-view stage. write_manager.on_catchup_finished() self._data.primaries = self._view_changer._primaries_selector.select_primaries( self._data.view_no) # ToDo: ugly way to understand node_reg changing self._previous_node_reg = self._write_manager.node_reg_handler.committed_node_reg bus.subscribe(Ordered, self.emulate_ordered_processing)
def _check_state(version): digest = taa_expected_digests[version] _digest = state.get( WriteRequestManager._state_path_taa_version(version), isCommitted=False) _data = state.get(WriteRequestManager._state_path_taa_digest(digest), isCommitted=False) if version in written: assert _digest == digest.encode() assert (config_state_serializer.deserialize(_data) == taa_expected_state_data[version]) else: assert _digest is None assert _data is None
def test_update_txn_author_agreement(taa_handler, write_manager, taa_input_data, taa_expected_state_data, taa_expected_digests): """ `update_txn_author_agreement` updates state properly """ state = write_manager.database_manager.get_state(CONFIG_LEDGER_ID) written = [] def _check_state(version): digest = taa_expected_digests[version] _digest = state.get( WriteRequestManager._state_path_taa_version(version), isCommitted=False) _data = state.get(WriteRequestManager._state_path_taa_digest(digest), isCommitted=False) if version in written: assert _digest == digest.encode() assert (config_state_serializer.deserialize(_data) == taa_expected_state_data[version]) else: assert _digest is None assert _data is None for data in taa_input_data: taa_handler._update_txn_author_agreement(data.text, data.version, data.seq_no, data.txn_time) written.append(data.version) digest = taa_expected_digests[data.version] assert state.get(WriteRequestManager._state_path_taa_latest(), isCommitted=False) == digest.encode() for version in taa_expected_state_data: _check_state(version)
def _check_state(version): expected_data = taa_aml_expected_data[version] _data = state.get( WriteRequestManager._state_path_taa_aml_version(version), isCommitted=False) if version in written: assert (config_state_serializer.deserialize(_data)[VALUE] == config_state_serializer.deserialize(expected_data)) else: assert _data is None
def fake_node(db_manager, fake_pool_cfg, fake_upgrader, fake_restarter, fake_pool_manager): wm = WriteRequestManager(database_manager=db_manager) rm = ReadRequestManager() return FakeSomething(write_manager=wm, read_manager=rm, db_manager=db_manager, write_req_validator=write_auth_req_validator, config=FakeSomething(stewardThreshold=20), poolCfg=fake_pool_cfg, upgrader=fake_upgrader, restarter=fake_restarter, poolManager=fake_pool_manager)
def write_req_manager(db): manager = WriteRequestManager(db) write_req_handler = NodeHandler(db, None) batch_req_handler = DomainBatchHandler(db) manager.register_req_handler(write_req_handler) manager.register_batch_handler(batch_req_handler) # We do not need to check request handler workability handler = manager.request_handlers[NODE][0] handler.static_validation = lambda request: 1 handler.dynamic_validation = lambda request: 1 handler.update_state = lambda txn, updated_state, request, is_committed: 1 handler.apply_request = lambda request, batch_ts, prev_result: (1, 1, 1) handler.apply_forced_request = lambda request: 1 handler.transform_txn_for_ledger = lambda txn: 1 # Same for batches handler = manager.batch_handlers[DOMAIN_LEDGER_ID][0] handler.post_batch_applied = lambda batch, prev_handler_result: 1 handler.commit_batch = lambda batch, prev_handler_result=None: 1 handler.post_batch_rejected = lambda ledger_id, prev_handler_result: 1 return manager
def write_manager(db_manager): wm = WriteRequestManager(db_manager) taa_handler = TxnAuthorAgreementHandler(db_manager) taa_aml_handler = TxnAuthorAgreementAmlHandler(db_manager) wm.register_req_handler(taa_handler) wm.register_req_handler(taa_aml_handler) return wm
def test_write_request_manager_restore_state( write_req_manager: WriteRequestManager, three_pc_batch, db): txn_type = "TXN_TYPE" ledger_id = 1 class MockHandler(WriteRequestHandler): def __init__(self): super().__init__(db, txn_type, ledger_id) self.last_update_state_call = None def static_validation(self, request: Request): pass def dynamic_validation(self, request: Request, req_pp_time: Optional[int]): pass def update_state(self, txn, prev_result, request, is_committed=False): self.last_update_state_call = txn, prev_result, request, is_committed handler_current = MockHandler() handler_prev_version = MockHandler() version = "version1" txn = { TXN_PAYLOAD: { TXN_PAYLOAD_TYPE: txn_type, TXN_VERSION: CURRENT_TXN_PAYLOAD_VERSIONS[txn_type] }, TXN_METADATA: { TXN_METADATA_TIME: 0 } } write_req_manager.register_req_handler(handler_current) write_req_manager.register_req_handler_with_version( handler_prev_version, version) write_req_manager.restore_state(txn, ledger_id) assert handler_current.last_update_state_call == (txn, None, None, True) assert handler_prev_version.last_update_state_call is None handler_current.last_update_state_call = None db.set_txn_version_controller( FakeSomething(get_pool_version=lambda a: version, update_version=lambda a: None)) write_req_manager.restore_state(txn, ledger_id) assert handler_current.last_update_state_call is None assert handler_prev_version.last_update_state_call == (txn, None, None, True)
def test_write_request_manager_chain_of_responsib_apply(write_req_manager: WriteRequestManager, node_req, db): write_req_manager.request_handlers[NODE] = [] handlers = write_req_manager.request_handlers[NODE] check_list = [FakeSomething(), FakeSomething(), FakeSomething()] node_req.check_list = check_list for check in check_list: check.check_field = False def modify_check_list(request, batch_ts, prev_result): assert not all(check.check_field for check in check_list) f_check = next(check for check in request.check_list if check.check_field is False) f_check.check_field = True return 1, 1, 1 for i in range(3): handler = NodeHandler(db, None) handler.apply_request = modify_check_list handlers.append(handler) write_req_manager.apply_request(node_req, 0) assert all(check.check_field for check in check_list)
def test_write_request_manager_handles_batches( write_req_manager: WriteRequestManager, three_pc_batch): write_req_manager.post_apply_batch(three_pc_batch) write_req_manager.commit_batch(three_pc_batch.ledger_id, len(three_pc_batch.valid_digests), three_pc_batch.state_root, three_pc_batch.txn_root, three_pc_batch.pp_time) write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
def test_write_request_manager_chain_of_responsib_batch( write_req_manager: WriteRequestManager, three_pc_batch, db): write_req_manager.batch_handlers[DOMAIN_LEDGER_ID] = [] handlers = write_req_manager.batch_handlers[DOMAIN_LEDGER_ID] check_list = [FakeSomething(), FakeSomething(), FakeSomething()] def modify_check_list(): assert not all(check.check_field for check in check_list) f_check = next(check for check in check_list if check.check_field is False) f_check.check_field = True def modify_check_list_post_apply(batch, prev_result): modify_check_list() return 1, 1, 1 def modify_check_list_commit(ledger_id, txn_count, state_root, txn_root, pp_time, prev_handler_result): modify_check_list() return 1, 1, 1 def modify_check_list_post_rejected(lid, prev_result): modify_check_list() return 1, 1, 1 for i in range(3): handler = DomainBatchHandler(db) handler.post_batch_applied = modify_check_list_post_apply handler.commit_batch = modify_check_list_commit handler.post_batch_rejected = modify_check_list_post_rejected handlers.append(handler) for check in check_list: check.check_field = False write_req_manager.post_apply_batch(three_pc_batch) assert all(check.check_field for check in check_list) for check in check_list: check.check_field = False write_req_manager.commit_batch(three_pc_batch.ledger_id, len(three_pc_batch.valid_digests), three_pc_batch.state_root, three_pc_batch.txn_root, three_pc_batch.pp_time) assert all(check.check_field for check in check_list) for check in check_list: check.check_field = False write_req_manager.post_batch_rejected(three_pc_batch.ledger_id) assert all(check.check_field for check in check_list)
def create_test_write_req_manager(name: str, genesis_txns: List) -> WriteRequestManager: db_manager = DatabaseManager() write_manager = WriteRequestManager(db_manager) read_manager = ReadRequestManager() bootstrap = TestLedgersBootstrap(write_req_manager=write_manager, read_req_manager=read_manager, action_req_manager=FakeSomething(), name=name, config=getConfig(), ledger_ids=Node.ledger_ids) bootstrap.set_genesis_transactions( [txn for txn in genesis_txns if get_type(txn) == NODE], [txn for txn in genesis_txns if get_type(txn) == NYM]) bootstrap.init() return write_manager
def test_write_request_manager_fails_to_handle_batches( write_req_manager: WriteRequestManager, three_pc_batch): nonexistent_lid = 999 three_pc_batch.ledger_id = nonexistent_lid with pytest.raises(LogicError): write_req_manager.post_apply_batch(three_pc_batch) with pytest.raises(LogicError): write_req_manager.post_batch_rejected(three_pc_batch) with pytest.raises(LogicError): write_req_manager.commit_batch(three_pc_batch)
def test_update_taa_aml(taa_aml_handler, taa_aml_input_data, taa_handler, write_manager, taa_aml_expected_state_data, taa_aml_expected_data): """ `update_txn_author_agreement` updates state properly """ state = write_manager.database_manager.get_state(CONFIG_LEDGER_ID) written = [] def _check_state(version): expected_data = taa_aml_expected_data[version] _data = state.get( WriteRequestManager._state_path_taa_aml_version(version), isCommitted=False) if version in written: assert (config_state_serializer.deserialize(_data)[VALUE] == config_state_serializer.deserialize(expected_data)) else: assert _data is None for data in taa_aml_input_data: taa_aml_handler._update_txn_author_agreement_acceptance_mechanisms( { AML_VERSION: data.version, AML: data.aml, AML_CONTEXT: data.amlContext }, data.seq_no, data.txn_time) written.append(data.version) data_d = data._asdict() assert state.get(WriteRequestManager._state_path_taa_aml_latest(), isCommitted=False) == encode_state_value( { AML_CONTEXT: data_d[AML_CONTEXT], AML: data_d[AML], AML_VERSION: data_d[AML_VERSION] }, data_d['seq_no'], data_d['txn_time']) for version in taa_aml_expected_state_data: _check_state(version)
def create_test_write_req_manager(name: str, genesis_txns: List) -> WriteRequestManager: db_manager = DatabaseManager() write_manager = WriteRequestManager(db_manager) read_manager = ReadRequestManager() register_test_handler(write_manager) db_manager.register_new_store(SEQ_NO_DB_LABEL, ReqIdrToTxn(KeyValueStorageInMemory())) bootstrap = TestLedgersBootstrap(write_req_manager=write_manager, read_req_manager=read_manager, action_req_manager=FakeSomething(), name=name, config=getConfig(), ledger_ids=Node.ledger_ids) bootstrap.set_genesis_transactions( [txn for txn in genesis_txns if get_type(txn) == NODE], [txn for txn in genesis_txns if get_type(txn) == NYM]) bootstrap.init() return write_manager
def test_write_request_manager_fails_to_handle_batches( write_req_manager: WriteRequestManager, three_pc_batch): nonexistent_lid = 999 three_pc_batch.ledger_id = nonexistent_lid with pytest.raises(LogicError): write_req_manager.post_apply_batch(three_pc_batch) with pytest.raises(LogicError): write_req_manager.post_batch_rejected(three_pc_batch) with pytest.raises(LogicError): write_req_manager.commit_batch(nonexistent_lid, len(three_pc_batch.valid_digests), three_pc_batch.state_root, three_pc_batch.txn_root, three_pc_batch.pp_time)
def fake_node(db_manager, fake_pool_cfg, fake_upgrader, fake_restarter, fake_pool_manager): wm = WriteRequestManager(database_manager=db_manager) rm = ReadRequestManager() am = ActionRequestManager() return FakeSomething( name="fake_node", dataLocation="//place_that_cannot_exist", genesis_dir="//place_that_cannot_exist", ledger_ids=Node.ledger_ids, write_manager=wm, read_manager=rm, action_manager=am, db_manager=db_manager, write_req_validator=write_auth_req_validator, config=FakeSomething( stewardThreshold=20, poolTransactionsFile="//pool_genesis_that_cannot_exist", domainTransactionsFile="//domain_genesis_that_cannot_exist"), poolCfg=fake_pool_cfg, upgrader=fake_upgrader, restarter=fake_restarter, poolManager=fake_pool_manager)
def __init__(self, name: str, validators: List[str], primary_name: str, timer: TimerService, bus: InternalBus, network: ExternalBus, write_manager: WriteRequestManager, bls_bft_replica: BlsBftReplica = None): # ToDo: Maybe ConsensusSharedData should be initiated before and passed already prepared? self._network = network self._data = ConsensusSharedData(name, validators, 0) self._data.primary_name = generateName(primary_name, self._data.inst_id) self._timer = timer self.config = getConfig() self.stasher = StashingRouter(self.config.REPLICA_STASH_LIMIT, buses=[bus, network]) self._write_manager = write_manager self._primaries_selector = RoundRobinNodeRegPrimariesSelector(self._write_manager.node_reg_handler) self._freshness_checker = FreshnessChecker(freshness_timeout=self.config.STATE_FRESHNESS_UPDATE_INTERVAL) for ledger_id in [POOL_LEDGER_ID, DOMAIN_LEDGER_ID, CONFIG_LEDGER_ID]: self._freshness_checker.register_ledger(ledger_id=ledger_id, initial_time=self.get_time_for_3pc_batch()) self._orderer = OrderingService(data=self._data, timer=self._timer, bus=bus, network=network, write_manager=self._write_manager, bls_bft_replica=bls_bft_replica, freshness_checker=self._freshness_checker, get_time_for_3pc_batch=self.get_time_for_3pc_batch, stasher=self.stasher) self._checkpointer = CheckpointService(self._data, bus, network, self.stasher, write_manager.database_manager) self._view_changer = ViewChangeService(self._data, self._timer, bus, network, self.stasher, self._primaries_selector) self._view_change_trigger = ViewChangeTriggerService(data=self._data, timer=self._timer, bus=bus, network=network, db_manager=write_manager.database_manager, is_master_degraded=lambda: False, stasher=self.stasher) self._primary_connection_monitor = PrimaryConnectionMonitorService(data=self._data, timer=self._timer, bus=bus, network=network) self._freshness_monitor = FreshnessMonitorService(data=self._data, timer=self._timer, bus=bus, network=network, freshness_checker=self._freshness_checker, get_time_for_3pc_batch=self.get_time_for_3pc_batch) self._message_requestor = MessageReqService(self._data, bus, network) self._add_ledgers() # TODO: This is just for testing purposes only self._data.checkpoints.append( Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw')) # ToDo: it should be done in Zero-view stage. write_manager.on_catchup_finished() # Simulate node behavior self._internal_bus = bus self._internal_bus.subscribe(NodeNeedViewChange, self.process_node_need_view_change) self._internal_bus.subscribe(Ordered, self.emulate_ordered_processing) # ToDo: ugly way to understand node_reg changing self._previous_node_reg = self._write_manager.node_reg_handler.committed_node_reg
def test_write_request_manager_handles_request( write_req_manager: WriteRequestManager, node_req): write_req_manager.static_validation(node_req) write_req_manager.dynamic_validation(node_req) write_req_manager.update_state(reqToTxn(node_req)) write_req_manager.apply_request(node_req, None) write_req_manager.apply_forced_request(node_req) write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))
def test_write_request_manager_fails_to_handle( write_req_manager: WriteRequestManager, node_req): node_req.operation['type'] = 999 with pytest.raises(LogicError): write_req_manager.static_validation(node_req) with pytest.raises(LogicError): write_req_manager.dynamic_validation(node_req) with pytest.raises(LogicError): write_req_manager.update_state(reqToTxn(node_req)) with pytest.raises(LogicError): write_req_manager.apply_request(node_req, None) with pytest.raises(LogicError): write_req_manager.apply_forced_request(node_req) with pytest.raises(LogicError): write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))
def test_write_request_manager_handles_batches( write_req_manager: WriteRequestManager, three_pc_batch): write_req_manager.post_apply_batch(three_pc_batch) write_req_manager.commit_batch(three_pc_batch) write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
def write_manager(db_manager): return WriteRequestManager(database_manager=db_manager)
def write_manager(db_manager): wrm = WriteRequestManager(database_manager=db_manager) wrm.node_reg_handler = NodeRegHandler(db_manager) return wrm
def test_state_path_taa_digest(): assert WriteRequestManager._state_path_taa_digest( 'some_digest') == b'2:d:some_digest'
def test_taa_digest(): assert WriteRequestManager._taa_digest('some_text', 'some_version') == \ "fb2ea9d28380a021ec747c442d62a68952b4b5813b45671098ad2b684b2f4646"
def test_state_path_taa_aml_version(): assert WriteRequestManager._state_path_taa_aml_version( 'some_version') == b'3:v:some_version'
def test_state_path_taa_latest(): assert WriteRequestManager._state_path_taa_latest() == b'2:latest'
def calc_taa_digest(text, version): return WriteRequestManager._taa_digest(text, version)