def __init__(self,
name: str,
validators: List[str],
primary_name: str,
timer: TimerService,
bus: InternalBus,
network: ExternalBus,
write_manager: WriteRequestManager,
bls_bft_replica: BlsBftReplica = None):
# ToDo: Maybe ConsensusSharedData should be initiated before and passed already prepared?
self._internal_bus = bus
self._data = ConsensusSharedData(name, validators, 0)
self._data.primary_name = generateName(primary_name,
self._data.inst_id)
self.config = getConfig()
self.stasher = StashingRouter(self.config.REPLICA_STASH_LIMIT,
buses=[bus, network])
self._write_manager = write_manager
self._primaries_selector = RoundRobinNodeRegPrimariesSelector(
self._write_manager.node_reg_handler)
self._orderer = OrderingService(
data=self._data,
timer=timer,
bus=bus,
network=network,
write_manager=self._write_manager,
bls_bft_replica=bls_bft_replica,
freshness_checker=FreshnessChecker(
freshness_timeout=self.config.STATE_FRESHNESS_UPDATE_INTERVAL),
primaries_selector=self._primaries_selector,
stasher=self.stasher)
self._checkpointer = CheckpointService(self._data, bus, network,
self.stasher,
write_manager.database_manager)
self._view_changer = ViewChangeService(self._data, timer, bus, network,
self.stasher,
self._primaries_selector)
self._message_requestor = MessageReqService(self._data, bus, network)
self._add_ledgers()
# TODO: This is just for testing purposes only
self._data.checkpoints.append(
Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw'))
# ToDo: it should be done in Zero-view stage.
write_manager.on_catchup_finished()
self._data.primaries = self._view_changer._primaries_selector.select_primaries(
self._data.view_no)
# ToDo: ugly way to understand node_reg changing
self._previous_node_reg = self._write_manager.node_reg_handler.committed_node_reg
bus.subscribe(Ordered, self.emulate_ordered_processing)
def _check_state(version):
digest = taa_expected_digests[version]
_digest = state.get(
WriteRequestManager._state_path_taa_version(version),
isCommitted=False)
_data = state.get(WriteRequestManager._state_path_taa_digest(digest),
isCommitted=False)
if version in written:
assert _digest == digest.encode()
assert (config_state_serializer.deserialize(_data) ==
taa_expected_state_data[version])
else:
assert _digest is None
assert _data is None
def test_update_txn_author_agreement(taa_handler, write_manager,
taa_input_data, taa_expected_state_data,
taa_expected_digests):
""" `update_txn_author_agreement` updates state properly """
state = write_manager.database_manager.get_state(CONFIG_LEDGER_ID)
written = []
def _check_state(version):
digest = taa_expected_digests[version]
_digest = state.get(
WriteRequestManager._state_path_taa_version(version),
isCommitted=False)
_data = state.get(WriteRequestManager._state_path_taa_digest(digest),
isCommitted=False)
if version in written:
assert _digest == digest.encode()
assert (config_state_serializer.deserialize(_data) ==
taa_expected_state_data[version])
else:
assert _digest is None
assert _data is None
for data in taa_input_data:
taa_handler._update_txn_author_agreement(data.text, data.version,
data.seq_no, data.txn_time)
written.append(data.version)
digest = taa_expected_digests[data.version]
assert state.get(WriteRequestManager._state_path_taa_latest(),
isCommitted=False) == digest.encode()
for version in taa_expected_state_data:
_check_state(version)
def _check_state(version):
expected_data = taa_aml_expected_data[version]
_data = state.get(
WriteRequestManager._state_path_taa_aml_version(version),
isCommitted=False)
if version in written:
assert (config_state_serializer.deserialize(_data)[VALUE] ==
config_state_serializer.deserialize(expected_data))
else:
assert _data is None
def fake_node(db_manager, fake_pool_cfg, fake_upgrader, fake_restarter, fake_pool_manager):
wm = WriteRequestManager(database_manager=db_manager)
rm = ReadRequestManager()
return FakeSomething(write_manager=wm,
read_manager=rm,
db_manager=db_manager,
write_req_validator=write_auth_req_validator,
config=FakeSomething(stewardThreshold=20),
poolCfg=fake_pool_cfg,
upgrader=fake_upgrader,
restarter=fake_restarter,
poolManager=fake_pool_manager)
def write_req_manager(db):
manager = WriteRequestManager(db)
write_req_handler = NodeHandler(db, None)
batch_req_handler = DomainBatchHandler(db)
manager.register_req_handler(write_req_handler)
manager.register_batch_handler(batch_req_handler)
# We do not need to check request handler workability
handler = manager.request_handlers[NODE][0]
handler.static_validation = lambda request: 1
handler.dynamic_validation = lambda request: 1
handler.update_state = lambda txn, updated_state, request, is_committed: 1
handler.apply_request = lambda request, batch_ts, prev_result: (1, 1, 1)
handler.apply_forced_request = lambda request: 1
handler.transform_txn_for_ledger = lambda txn: 1
# Same for batches
handler = manager.batch_handlers[DOMAIN_LEDGER_ID][0]
handler.post_batch_applied = lambda batch, prev_handler_result: 1
handler.commit_batch = lambda batch, prev_handler_result=None: 1
handler.post_batch_rejected = lambda ledger_id, prev_handler_result: 1
return manager
def write_manager(db_manager):
wm = WriteRequestManager(db_manager)
taa_handler = TxnAuthorAgreementHandler(db_manager)
taa_aml_handler = TxnAuthorAgreementAmlHandler(db_manager)
wm.register_req_handler(taa_handler)
wm.register_req_handler(taa_aml_handler)
return wm
def test_write_request_manager_restore_state(
write_req_manager: WriteRequestManager, three_pc_batch, db):
txn_type = "TXN_TYPE"
ledger_id = 1
class MockHandler(WriteRequestHandler):
def __init__(self):
super().__init__(db, txn_type, ledger_id)
self.last_update_state_call = None
def static_validation(self, request: Request):
pass
def dynamic_validation(self, request: Request,
req_pp_time: Optional[int]):
pass
def update_state(self, txn, prev_result, request, is_committed=False):
self.last_update_state_call = txn, prev_result, request, is_committed
handler_current = MockHandler()
handler_prev_version = MockHandler()
version = "version1"
txn = {
TXN_PAYLOAD: {
TXN_PAYLOAD_TYPE: txn_type,
TXN_VERSION: CURRENT_TXN_PAYLOAD_VERSIONS[txn_type]
},
TXN_METADATA: {
TXN_METADATA_TIME: 0
}
}
write_req_manager.register_req_handler(handler_current)
write_req_manager.register_req_handler_with_version(
handler_prev_version, version)
write_req_manager.restore_state(txn, ledger_id)
assert handler_current.last_update_state_call == (txn, None, None, True)
assert handler_prev_version.last_update_state_call is None
handler_current.last_update_state_call = None
db.set_txn_version_controller(
FakeSomething(get_pool_version=lambda a: version,
update_version=lambda a: None))
write_req_manager.restore_state(txn, ledger_id)
assert handler_current.last_update_state_call is None
assert handler_prev_version.last_update_state_call == (txn, None, None,
True)
def test_write_request_manager_chain_of_responsib_apply(write_req_manager: WriteRequestManager,
node_req, db):
write_req_manager.request_handlers[NODE] = []
handlers = write_req_manager.request_handlers[NODE]
check_list = [FakeSomething(), FakeSomething(), FakeSomething()]
node_req.check_list = check_list
for check in check_list:
check.check_field = False
def modify_check_list(request, batch_ts, prev_result):
assert not all(check.check_field for check in check_list)
f_check = next(check for check in request.check_list if check.check_field is False)
f_check.check_field = True
return 1, 1, 1
for i in range(3):
handler = NodeHandler(db, None)
handler.apply_request = modify_check_list
handlers.append(handler)
write_req_manager.apply_request(node_req, 0)
assert all(check.check_field for check in check_list)
def test_write_request_manager_handles_batches(
write_req_manager: WriteRequestManager, three_pc_batch):
write_req_manager.post_apply_batch(three_pc_batch)
write_req_manager.commit_batch(three_pc_batch.ledger_id,
len(three_pc_batch.valid_digests),
three_pc_batch.state_root,
three_pc_batch.txn_root,
three_pc_batch.pp_time)
write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
def test_write_request_manager_chain_of_responsib_batch(
write_req_manager: WriteRequestManager, three_pc_batch, db):
write_req_manager.batch_handlers[DOMAIN_LEDGER_ID] = []
handlers = write_req_manager.batch_handlers[DOMAIN_LEDGER_ID]
check_list = [FakeSomething(), FakeSomething(), FakeSomething()]
def modify_check_list():
assert not all(check.check_field for check in check_list)
f_check = next(check for check in check_list
if check.check_field is False)
f_check.check_field = True
def modify_check_list_post_apply(batch, prev_result):
modify_check_list()
return 1, 1, 1
def modify_check_list_commit(ledger_id, txn_count, state_root, txn_root,
pp_time, prev_handler_result):
modify_check_list()
return 1, 1, 1
def modify_check_list_post_rejected(lid, prev_result):
modify_check_list()
return 1, 1, 1
for i in range(3):
handler = DomainBatchHandler(db)
handler.post_batch_applied = modify_check_list_post_apply
handler.commit_batch = modify_check_list_commit
handler.post_batch_rejected = modify_check_list_post_rejected
handlers.append(handler)
for check in check_list:
check.check_field = False
write_req_manager.post_apply_batch(three_pc_batch)
assert all(check.check_field for check in check_list)
for check in check_list:
check.check_field = False
write_req_manager.commit_batch(three_pc_batch.ledger_id,
len(three_pc_batch.valid_digests),
three_pc_batch.state_root,
three_pc_batch.txn_root,
three_pc_batch.pp_time)
assert all(check.check_field for check in check_list)
for check in check_list:
check.check_field = False
write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
assert all(check.check_field for check in check_list)
def create_test_write_req_manager(name: str,
genesis_txns: List) -> WriteRequestManager:
db_manager = DatabaseManager()
write_manager = WriteRequestManager(db_manager)
read_manager = ReadRequestManager()
bootstrap = TestLedgersBootstrap(write_req_manager=write_manager,
read_req_manager=read_manager,
action_req_manager=FakeSomething(),
name=name,
config=getConfig(),
ledger_ids=Node.ledger_ids)
bootstrap.set_genesis_transactions(
[txn for txn in genesis_txns if get_type(txn) == NODE],
[txn for txn in genesis_txns if get_type(txn) == NYM])
bootstrap.init()
return write_manager
def test_write_request_manager_fails_to_handle_batches(
write_req_manager: WriteRequestManager, three_pc_batch):
nonexistent_lid = 999
three_pc_batch.ledger_id = nonexistent_lid
with pytest.raises(LogicError):
write_req_manager.post_apply_batch(three_pc_batch)
with pytest.raises(LogicError):
write_req_manager.post_batch_rejected(three_pc_batch)
with pytest.raises(LogicError):
write_req_manager.commit_batch(three_pc_batch)
def test_update_taa_aml(taa_aml_handler, taa_aml_input_data, taa_handler,
write_manager, taa_aml_expected_state_data,
taa_aml_expected_data):
""" `update_txn_author_agreement` updates state properly """
state = write_manager.database_manager.get_state(CONFIG_LEDGER_ID)
written = []
def _check_state(version):
expected_data = taa_aml_expected_data[version]
_data = state.get(
WriteRequestManager._state_path_taa_aml_version(version),
isCommitted=False)
if version in written:
assert (config_state_serializer.deserialize(_data)[VALUE] ==
config_state_serializer.deserialize(expected_data))
else:
assert _data is None
for data in taa_aml_input_data:
taa_aml_handler._update_txn_author_agreement_acceptance_mechanisms(
{
AML_VERSION: data.version,
AML: data.aml,
AML_CONTEXT: data.amlContext
}, data.seq_no, data.txn_time)
written.append(data.version)
data_d = data._asdict()
assert state.get(WriteRequestManager._state_path_taa_aml_latest(),
isCommitted=False) == encode_state_value(
{
AML_CONTEXT: data_d[AML_CONTEXT],
AML: data_d[AML],
AML_VERSION: data_d[AML_VERSION]
}, data_d['seq_no'], data_d['txn_time'])
for version in taa_aml_expected_state_data:
_check_state(version)
def create_test_write_req_manager(name: str,
genesis_txns: List) -> WriteRequestManager:
db_manager = DatabaseManager()
write_manager = WriteRequestManager(db_manager)
read_manager = ReadRequestManager()
register_test_handler(write_manager)
db_manager.register_new_store(SEQ_NO_DB_LABEL,
ReqIdrToTxn(KeyValueStorageInMemory()))
bootstrap = TestLedgersBootstrap(write_req_manager=write_manager,
read_req_manager=read_manager,
action_req_manager=FakeSomething(),
name=name,
config=getConfig(),
ledger_ids=Node.ledger_ids)
bootstrap.set_genesis_transactions(
[txn for txn in genesis_txns if get_type(txn) == NODE],
[txn for txn in genesis_txns if get_type(txn) == NYM])
bootstrap.init()
return write_manager
def test_write_request_manager_fails_to_handle_batches(
write_req_manager: WriteRequestManager, three_pc_batch):
nonexistent_lid = 999
three_pc_batch.ledger_id = nonexistent_lid
with pytest.raises(LogicError):
write_req_manager.post_apply_batch(three_pc_batch)
with pytest.raises(LogicError):
write_req_manager.post_batch_rejected(three_pc_batch)
with pytest.raises(LogicError):
write_req_manager.commit_batch(nonexistent_lid,
len(three_pc_batch.valid_digests),
three_pc_batch.state_root,
three_pc_batch.txn_root,
three_pc_batch.pp_time)
def fake_node(db_manager, fake_pool_cfg, fake_upgrader, fake_restarter,
fake_pool_manager):
wm = WriteRequestManager(database_manager=db_manager)
rm = ReadRequestManager()
am = ActionRequestManager()
return FakeSomething(
name="fake_node",
dataLocation="//place_that_cannot_exist",
genesis_dir="//place_that_cannot_exist",
ledger_ids=Node.ledger_ids,
write_manager=wm,
read_manager=rm,
action_manager=am,
db_manager=db_manager,
write_req_validator=write_auth_req_validator,
config=FakeSomething(
stewardThreshold=20,
poolTransactionsFile="//pool_genesis_that_cannot_exist",
domainTransactionsFile="//domain_genesis_that_cannot_exist"),
poolCfg=fake_pool_cfg,
upgrader=fake_upgrader,
restarter=fake_restarter,
poolManager=fake_pool_manager)
def __init__(self, name: str, validators: List[str], primary_name: str,
timer: TimerService, bus: InternalBus, network: ExternalBus,
write_manager: WriteRequestManager,
bls_bft_replica: BlsBftReplica = None):
# ToDo: Maybe ConsensusSharedData should be initiated before and passed already prepared?
self._network = network
self._data = ConsensusSharedData(name, validators, 0)
self._data.primary_name = generateName(primary_name, self._data.inst_id)
self._timer = timer
self.config = getConfig()
self.stasher = StashingRouter(self.config.REPLICA_STASH_LIMIT, buses=[bus, network])
self._write_manager = write_manager
self._primaries_selector = RoundRobinNodeRegPrimariesSelector(self._write_manager.node_reg_handler)
self._freshness_checker = FreshnessChecker(freshness_timeout=self.config.STATE_FRESHNESS_UPDATE_INTERVAL)
for ledger_id in [POOL_LEDGER_ID, DOMAIN_LEDGER_ID, CONFIG_LEDGER_ID]:
self._freshness_checker.register_ledger(ledger_id=ledger_id,
initial_time=self.get_time_for_3pc_batch())
self._orderer = OrderingService(data=self._data,
timer=self._timer,
bus=bus,
network=network,
write_manager=self._write_manager,
bls_bft_replica=bls_bft_replica,
freshness_checker=self._freshness_checker,
get_time_for_3pc_batch=self.get_time_for_3pc_batch,
stasher=self.stasher)
self._checkpointer = CheckpointService(self._data, bus, network, self.stasher,
write_manager.database_manager)
self._view_changer = ViewChangeService(self._data, self._timer, bus, network, self.stasher, self._primaries_selector)
self._view_change_trigger = ViewChangeTriggerService(data=self._data,
timer=self._timer,
bus=bus,
network=network,
db_manager=write_manager.database_manager,
is_master_degraded=lambda: False,
stasher=self.stasher)
self._primary_connection_monitor = PrimaryConnectionMonitorService(data=self._data,
timer=self._timer,
bus=bus,
network=network)
self._freshness_monitor = FreshnessMonitorService(data=self._data,
timer=self._timer,
bus=bus,
network=network,
freshness_checker=self._freshness_checker,
get_time_for_3pc_batch=self.get_time_for_3pc_batch)
self._message_requestor = MessageReqService(self._data, bus, network)
self._add_ledgers()
# TODO: This is just for testing purposes only
self._data.checkpoints.append(
Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0,
digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw'))
# ToDo: it should be done in Zero-view stage.
write_manager.on_catchup_finished()
# Simulate node behavior
self._internal_bus = bus
self._internal_bus.subscribe(NodeNeedViewChange, self.process_node_need_view_change)
self._internal_bus.subscribe(Ordered, self.emulate_ordered_processing)
# ToDo: ugly way to understand node_reg changing
self._previous_node_reg = self._write_manager.node_reg_handler.committed_node_reg
def test_write_request_manager_handles_request(
write_req_manager: WriteRequestManager, node_req):
write_req_manager.static_validation(node_req)
write_req_manager.dynamic_validation(node_req)
write_req_manager.update_state(reqToTxn(node_req))
write_req_manager.apply_request(node_req, None)
write_req_manager.apply_forced_request(node_req)
write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))
def test_write_request_manager_fails_to_handle(
write_req_manager: WriteRequestManager, node_req):
node_req.operation['type'] = 999
with pytest.raises(LogicError):
write_req_manager.static_validation(node_req)
with pytest.raises(LogicError):
write_req_manager.dynamic_validation(node_req)
with pytest.raises(LogicError):
write_req_manager.update_state(reqToTxn(node_req))
with pytest.raises(LogicError):
write_req_manager.apply_request(node_req, None)
with pytest.raises(LogicError):
write_req_manager.apply_forced_request(node_req)
with pytest.raises(LogicError):
write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))
def test_write_request_manager_handles_batches(
write_req_manager: WriteRequestManager, three_pc_batch):
write_req_manager.post_apply_batch(three_pc_batch)
write_req_manager.commit_batch(three_pc_batch)
write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
def write_manager(db_manager):
return WriteRequestManager(database_manager=db_manager)
def write_manager(db_manager):
wrm = WriteRequestManager(database_manager=db_manager)
wrm.node_reg_handler = NodeRegHandler(db_manager)
return wrm
def test_state_path_taa_digest():
assert WriteRequestManager._state_path_taa_digest(
'some_digest') == b'2:d:some_digest'
def test_taa_digest():
assert WriteRequestManager._taa_digest('some_text', 'some_version') == \
"fb2ea9d28380a021ec747c442d62a68952b4b5813b45671098ad2b684b2f4646"
def test_state_path_taa_aml_version():
assert WriteRequestManager._state_path_taa_aml_version(
'some_version') == b'3:v:some_version'
def test_state_path_taa_latest():
assert WriteRequestManager._state_path_taa_latest() == b'2:latest'
def calc_taa_digest(text, version):
return WriteRequestManager._taa_digest(text, version)
