class TestIntegrateServiceConfigurationInitial(TestIntegrateBase): def setUp(self): self._block_height = 0 self._prev_block_hash = None self.config = IconConfig("", default_icon_config) self.config.load() self.config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin.address)}) self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False}}) self.config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) def test_service_configuration_fee_setting(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_FEE: True}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) context = IconScoreContext(IconScoreContextType.INVOKE) self.assertEqual(context.icon_service_flag, IconServiceFlag.FEE) def test_service_configuration_audit_setting(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: True}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) context = IconScoreContext(IconScoreContextType.INVOKE) self.assertEqual(context.icon_service_flag, IconServiceFlag.AUDIT) def test_service_configuration_deployer_white_list_setting(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: True}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) context = IconScoreContext(IconScoreContextType.INVOKE) self.assertEqual(context.icon_service_flag, IconServiceFlag.DEPLOYER_WHITE_LIST) def test_service_configuration_score_package_validiator_setting(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: True}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) context = IconScoreContext(IconScoreContextType.INVOKE) self.assertEqual(context.icon_service_flag, IconServiceFlag.SCORE_PACKAGE_VALIDATOR) def test_service_configuration_multiple_setting(self): multiple_config = {ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: True, ConfigKey.SERVICE_FEE: True, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: True, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: True}} self.config.update_conf(multiple_config) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) context = IconScoreContext(IconScoreContextType.INVOKE) expected_flag = IconServiceFlag.FEE | IconServiceFlag.AUDIT | \ IconServiceFlag.SCORE_PACKAGE_VALIDATOR | IconServiceFlag.DEPLOYER_WHITE_LIST self.assertEqual(context.icon_service_flag, expected_flag)
def generate_service_engine(db_factory_create_by_name, icx_engine_open): service_engine = IconServiceEngine() service_engine._load_builtin_scores = Mock() # Mocks _init_global_value_by_governance_score # to ignore initializing governance SCORE service_engine._init_global_value_by_governance_score = Mock() service_engine.open(IconConfig("", default_icon_config)) # Patches create_by_name to pass creating DB db_factory_create_by_name.assert_called() icx_engine_open.assert_called() service_engine._load_builtin_scores.assert_called() service_engine._init_global_value_by_governance_score.assert_called() # Ignores icx transfer service_engine._icx_engine._transfer = Mock() # Mocks get_balance so, it returns always 100 icx service_engine._icx_engine.get_balance = Mock(return_value=100 * 10**18) return service_engine
def setUp(self): self._state_db_root_path = '.db' self._score_root_path = '.score' rmtree(self._score_root_path) rmtree(self._state_db_root_path) engine = IconServiceEngine() conf = IconConfig("", default_icon_config) conf.load() conf.update_conf({ ConfigKey.BUILTIN_SCORE_OWNER: str(create_address(AddressPrefix.EOA)), ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) # engine._load_builtin_scores = Mock() # engine._init_global_value_by_governance_score = Mock() engine.open(conf) self._engine = engine self._genesis_address = create_address(AddressPrefix.EOA) self._treasury_address = create_address(AddressPrefix.EOA) self._governance_score_address =\ Address.from_string('cx0000000000000000000000000000000000000001') self.from_ = self._genesis_address self._to = create_address(AddressPrefix.EOA) self._icon_score_address = create_address(AddressPrefix.CONTRACT) self._total_supply = 100 * 10**18 accounts = [{ 'name': 'god', 'address': self._genesis_address, 'balance': self._total_supply }, { 'name': 'treasury', 'address': self._treasury_address, 'balance': 0 }] block = Block(0, create_block_hash(), 0, None) tx = { 'method': '', 'params': { 'txHash': create_tx_hash() }, 'genesisData': { 'accounts': accounts } } tx_lists = [tx] self._engine.invoke(block, tx_lists) self._engine.commit(block) self.genesis_block = block
def _create_service_engine( rc_db_from_path, db_factory_create_by_name, icx_engine_open, icx_storage_open, iiss_engine_open, iiss_storage_open, prep_engine_open): service_engine = IconServiceEngine() service_engine._load_builtin_scores = Mock() # Mocks _init_global_value_by_governance_score # to ignore initializing governance SCORE service_engine._init_global_value_by_governance_score = Mock() state_db = {} rc_db = {} def state_put(self, key, value): state_db[key] = value def state_get(self, key): return state_db.get(key) def rc_put(key, value): rc_db[key] = value def rc_get(key): return rc_db.get(key) context_db = Mock(spec=ContextDatabase) context_db.key_value_db = state_db context_db.get = state_get context_db.put = state_put iiss_mock_db = Mock(spec=KeyValueDatabase) iiss_mock_db.get = rc_get iiss_mock_db.put = rc_put db_factory_create_by_name.return_value = context_db rc_db_from_path.return_value = iiss_mock_db service_engine.open(IconConfig("", default_icon_config)) # Patches create_by_name to pass creating DB rc_db_from_path.assert_called() db_factory_create_by_name.assert_called() icx_engine_open.assert_called() service_engine._load_builtin_scores.assert_called() service_engine._init_global_value_by_governance_score.assert_called() service_engine._icon_pre_validator._is_inactive_score = Mock() return service_engine
class TestIntegrateBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.score' cls._state_db_root_path = '.statedb' cls._test_sample_root = "" cls._signature = "VAia7YZ2Ji6igKWzjR2YsGa2m53nKPrfK7uXYW78QLE+ATehAVZPC40szvAiA6NEU5gCYB4c4qaQzqDh2ugcHgA=" cls._version = 3 cls._step_limit = 1 * 10**9 cls._icx_factor = 10**18 cls.admin: 'Address' = create_address() cls.genesis: 'Address' = create_address() cls.owner1: 'Address' = create_address() cls.owner2: 'Address' = create_address() cls.owner3: 'Address' = create_address() cls._fee_treasury: 'Address' = create_address() def setUp(self): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self.admin)}) config.update_conf({ ConfigKey.SERVICE: { ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False } }) config.update_conf({ ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke() def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} def _genesis_invoke(self) -> list: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': self._version, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [ { "name": "genesis", "address": self.genesis, "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": self._fee_treasury, "balance": 0 }, { "name": "owner1", "address": self.owner1, "balance": 100 * self._icx_factor }, { "name": "owner2", "address": self.owner2, "balance": 100 * self._icx_factor }, { "name": "owner3", "address": self.owner3, "balance": 100 * self._icx_factor }, ] }, } block_hash = create_block_hash() block = Block(self._block_height, block_hash, timestamp_us, None) invoke_response, state_root_hash, added_transactions, next_preps = \ self.icon_service_engine.invoke(block=block, tx_requests=[tx]) self.icon_service_engine.commit(block.height, block.hash, None) self._block_height += 1 self._prev_block_hash = block_hash return invoke_response def deploy_score(self, package_name: str, deployer_address: 'Address', deploy_params: dict = None): tx1 = self._make_deploy_tx("", package_name, deployer_address, ZERO_SCORE_ADDRESS, deploy_params=deploy_params) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) score_address = tx_results[0].score_address return score_address def _make_deploy_tx(self, score_root: str, score_name: str, addr_from: Union['Address', None], addr_to: 'Address', deploy_params: dict = None, timestamp_us: int = None, data: bytes = None, is_sys: bool = False): if deploy_params is None: deploy_params = {} score_path = get_score_path(score_root, score_name) if is_sys: deploy_data = { 'contentType': 'application/tbears', 'content': score_path, 'params': deploy_params } else: if data is None: mz = InMemoryZip() mz.zip_in_memory(score_path) data = f'0x{mz.data.hex()}' else: data = f'0x{bytes.hex(data)}' deploy_data = { 'contentType': 'application/zip', 'content': data, 'params': deploy_params } if timestamp_us is None: timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "deploy", "data": deploy_data } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} self.icon_service_engine.validate_transaction(tx) return tx def query(self, score_address: 'Address', method: str, params: dict = None): params = {} if params is None else params query_request = { "version": self._version, "from": self.admin, "to": score_address, "dataType": "call", "data": { "method": method, "params": params } } return self._query(query_request) def send_tx(self, addr_from: Optional['Address'], addr_to: 'Address', method: str, params: dict, value: int = 0): tx = self._make_score_call_tx(addr_from, addr_to, method, params, value) prev_block, tx_results = self._make_and_req_block([tx]) self.assertEqual(int(True), tx_results[0].status) self._write_precommit_state(prev_block) return tx_results[0] def send_message(self, addr_from: Optional['Address'], addr_to: 'Address', data: str, value: int = 0): tx = self._make_score_message_tx(addr_from, addr_to, data, value) prev_block, tx_results = self._make_and_req_block([tx]) self.assertEqual(int(True), tx_results[0].status) self._write_precommit_state(prev_block) return tx_results[0] def _make_score_message_tx(self, addr_from: Optional['Address'], addr_to: 'Address', data: str, value: int = 0): timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "value": value, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "message", "data": data } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} self.icon_service_engine.validate_transaction(tx) return tx def _make_score_call_tx(self, addr_from: Optional['Address'], addr_to: 'Address', method: str, params: dict, value: int = 0): timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "value": value, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "call", "data": { "method": method, "params": params } } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} self.icon_service_engine.validate_transaction(tx) return tx def icx_send_tx(self, addr_from: Optional['Address'], addr_to: Union['Address', 'MalformedAddress'], value: int, disable_pre_validate: bool = False, support_v2: bool = False): tx = self._make_icx_send_tx(addr_from, addr_to, value, support_v2) prev_block, tx_results = self._make_and_req_block([tx]) self.assertEqual(tx_results[0].status, int(True)) self._write_precommit_state(prev_block) return tx_results def _make_icx_send_tx(self, addr_from: Optional['Address'], addr_to: Union['Address', 'MalformedAddress'], value: int, disable_pre_validate: bool = False, support_v2: bool = False): timestamp_us = create_timestamp() nonce = 0 request_params = { "from": addr_from, "to": addr_to, "value": value, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature } if support_v2: request_params["fee"] = 10**16 else: request_params["version"] = self._version method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} if not disable_pre_validate: self.icon_service_engine.validate_transaction(tx) return tx def _make_and_req_block(self, tx_list: list, block_height: int = None) -> tuple: if block_height is None: block_height: int = self._block_height block_hash = create_block_hash() timestamp_us = create_timestamp() block = Block(block_height, block_hash, timestamp_us, self._prev_block_hash) tx_results, state_root_hash, added_transactions, next_preps = \ self.icon_service_engine.invoke(block=block, tx_requests=tx_list) return block, tx_results def _write_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.commit(block.height, block.hash, None) self._block_height += 1 self._prev_block_hash = block.hash def _remove_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.rollback(block.height, block.hash) def _query(self, request: dict, method: str = 'icx_call') -> Any: response = self.icon_service_engine.query(method, request) return response def _create_invalid_block(self, block_height: int = None) -> 'Block': if block_height is None: block_height: int = self._block_height block_hash = create_block_hash() timestamp_us = create_timestamp() return Block(block_height, block_hash, timestamp_us, self._prev_block_hash)
class IconIntegrateTestBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.testscore' cls._state_db_root_path = '.teststatedb' cls._icx_factor = 10**18 cls._genesis: 'KeyWallet' = KeyWallet.create() cls._fee_treasury: 'KeyWallet' = KeyWallet.create() cls._test1: 'KeyWallet' = KeyWallet.load( bytes.fromhex(TEST1_PRIVATE_KEY)) cls._wallet_array = [KeyWallet.load(v) for v in TEST_ACCOUNTS] def setUp(self, genesis_accounts: List[Account] = None, block_confirm_interval: int = tbears_server_config[ TbConf.BLOCK_CONFIRM_INTERVAL], network_only: bool = False, network_delay_ms: int = tbears_server_config[ TbConf.NETWORK_DELAY_MS]): self._block_height = -1 self._prev_block_hash = None self._block_confirm_interval = block_confirm_interval self._network_only: bool = network_only self._network_delay: float = network_delay_ms / 1000 if self._network_only: return root_clear(self._score_root_path, self._state_db_root_path) config = IconConfig("", default_icon_config) config.load() config.update_conf( {ConfigKey.BUILTIN_SCORE_OWNER: self._test1.get_address()}) config.update_conf({ ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self._mock_rc_proxy() self.icon_service_engine.open(config) self._genesis_invoke(genesis_accounts) self._tx_results: dict = {} def tearDown(self): if not self._network_only: self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} @staticmethod def _append_list(tx: dict, genesis_accounts: List[Account]) -> None: """Appends additional genesis account list to genesisData :param genesis_accounts: additional genesis account list consisted of namedtuple named Account of which keys are name, address and balance :return: None """ for account_as_namedtuple in genesis_accounts: tx["genesisData"]['accounts'].append({ "name": account_as_namedtuple.name, "address": account_as_namedtuple.address, "balance": account_as_namedtuple.balance }) def _genesis_invoke(self, genesis_accounts: List[Account]) -> list: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': 3, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [{ "name": "genesis", "address": Address.from_string(self._genesis.get_address()), "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": Address.from_string(self._fee_treasury.get_address()), "balance": 0 }, { "name": "_admin", "address": Address.from_string(self._test1.get_address()), "balance": 1_000_000 * self._icx_factor }] }, }
class IconScoreInnerTask(object): def __init__(self, conf: 'IconConfig'): self._conf = conf self._thread_flag = ENABLE_THREAD_FLAG self._icon_service_engine = IconServiceEngine() self._open() self._thread_pool = { THREAD_INVOKE: ThreadPoolExecutor(1), THREAD_QUERY: ThreadPoolExecutor(1), THREAD_VALIDATE: ThreadPoolExecutor(1) } def _open(self): Logger.info("icon_score_service open", ICON_INNER_LOG_TAG) self._icon_service_engine.open(self._conf) def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool: return (self._thread_flag & flag) == flag def _log_exception(self, e: BaseException, tag: str = ICON_INNER_LOG_TAG) -> None: Logger.exception(e, tag) Logger.error(e, tag) @message_queue_task async def hello(self): Logger.info('icon_score_hello', ICON_INNER_LOG_TAG) def _close(self): Logger.info("icon_score_service close", ICON_INNER_LOG_TAG) if self._icon_service_engine: self._icon_service_engine.close() self._icon_service_engine = None MessageQueueService.loop.stop() @message_queue_task async def close(self): self._close() @message_queue_task async def invoke(self, request: dict): Logger.info(f'invoke request with {request}', ICON_INNER_LOG_TAG) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._invoke, request) else: return self._invoke(request) def _invoke(self, request: dict): """Process transactions in a block :param request: :return: """ response = None try: params = TypeConverter.convert(request, ParamType.INVOKE) converted_block_params = params['block'] block = Block.from_dict(converted_block_params) converted_tx_requests = params['transactions'] tx_results, state_root_hash = self._icon_service_engine.invoke( block=block, tx_requests=converted_tx_requests) convert_tx_results = \ {bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case) for tx_result in tx_results} results = { 'txResults': convert_tx_results, 'stateRootHash': bytes.hex(state_root_hash) } response = MakeResponse.make_response(results) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SERVER_ERROR, str(e)) finally: Logger.info(f'invoke response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def query(self, request: dict): Logger.info(f'query request with {request}', ICON_INNER_LOG_TAG) if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._query, request) else: return self._query(request) def _query(self, request: dict): response = None try: method = request['method'] if method == 'debug_estimateStep': converted_request = TypeConverter.convert( request, ParamType.INVOKE_TRANSACTION) value = self._icon_service_engine.estimate_step( converted_request) else: converted_request = TypeConverter.convert( request, ParamType.QUERY) value = self._icon_service_engine.query( method, converted_request['params']) if isinstance(value, Address): value = str(value) response = MakeResponse.make_response(value) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SERVER_ERROR, str(e)) finally: Logger.info(f'query response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def write_precommit_state(self, request: dict): Logger.info(f'write_precommit_state request with {request}', ICON_INNER_LOG_TAG) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._write_precommit_state, request) else: return self._write_precommit_state(request) def _write_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block = Block.from_dict(converted_block_params) self._icon_service_engine.commit(block) response = MakeResponse.make_response(ExceptionCode.OK) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SERVER_ERROR, str(e)) finally: Logger.info(f'write_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def remove_precommit_state(self, request: dict): Logger.info(f'remove_precommit_state request with {request}', ICON_INNER_LOG_TAG) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._remove_precommit_state, request) else: return self._remove_precommit_state(request) def _remove_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block = Block.from_dict(converted_block_params) self._icon_service_engine.rollback(block) response = MakeResponse.make_response(ExceptionCode.OK) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SERVER_ERROR, str(e)) finally: Logger.info(f'remove_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def validate_transaction(self, request: dict): Logger.info(f'pre_validate_check request with {request}', ICON_INNER_LOG_TAG) if self._is_thread_flag_on(EnableThreadFlag.VALIDATE): loop = get_event_loop() return await loop.run_in_executor( self._thread_pool[THREAD_VALIDATE], self._validate_transaction, request) else: return self._validate_transaction(request) def _validate_transaction(self, request: dict): response = None try: converted_request = TypeConverter.convert( request, ParamType.VALIDATE_TRANSACTION) self._icon_service_engine.validate_transaction(converted_request) response = MakeResponse.make_response(ExceptionCode.OK) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SERVER_ERROR, str(e)) finally: Logger.info(f'pre_validate_check response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def change_block_hash(self, params): return ExceptionCode.OK
class TestIntegrateExistentScoresAudit(TestIntegrateBase): # override setUp method for making directory before begin tests. def setUp(self): root_clear(self._score_root_path, self._state_db_root_path, self._iiss_db_root_path, self._precommit_log_path) self._block_height = -1 self._prev_block_hash = None self.config = IconConfig("", default_icon_config) self.config.load() self.config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin.address)}) self.config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) self.config.update_conf(self._make_init_config()) def _setUp_audit(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: True, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) self._genesis_invoke() self.token_initial_params = {"init_supply": hex(1000), "decimal": "0x12"} def _make_directories_in_builtin_score_path(self): for score_name, address in BUILTIN_SCORE_ADDRESS_MAPPER.items(): os.makedirs(os.path.join(self._score_root_path, f'01{address[2:]}', f"0x{'0' * 64}"), exist_ok=True) def _make_directory_using_address_and_hash(self, score_address: 'Address', tx_hash: bytes): tx_str = f"0x{bytes.hex(tx_hash)}" os.makedirs(os.path.join(self._score_root_path, f"01{str(score_address)[2:]}", tx_str)) def _create_deploy_score_tx(self, score_root: str, score_name: str, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address'], deploy_params: dict) -> dict: addr_from: Optional['Address'] = self._convert_address_from_address_type(from_) addr_to: 'Address' = self._convert_address_from_address_type(to_) timestamp = int(time.time() * 10 ** 6) tx: dict = self.create_deploy_score_tx(score_root=score_root, score_name=score_name, from_=addr_from, to_=addr_to, deploy_params=deploy_params, timestamp_us=timestamp) score_address: 'Address' = generate_score_address(addr_from, timestamp, nonce=0) tx_hash: bytes = tx['params']['txHash'] self._make_directory_using_address_and_hash(score_address, tx_hash) return tx def _deploy_score(self, score_root: str, score_name: str, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address'], deploy_params: dict, expected_status: bool = True) -> List['TransactionResult']: tx: dict = self._create_deploy_score_tx(score_root=score_root, score_name=score_name, from_=from_, to_=to_, deploy_params=deploy_params) tx_results: List['TransactionResult'] = self.process_confirm_block_tx([tx], expected_status) return tx_results def test_existent_builtin_score_audit(self): self._make_directories_in_builtin_score_path() self._setUp_audit() # original SCORE api original_governance_api: dict = self.get_score_api(GOVERNANCE_SCORE_ADDRESS) # update governance SCORE(revision2) tx_results: List['TransactionResult'] = self.update_governance("0_0_4") self.accept_score(tx_results[0].tx_hash) # updated SCORE api updated_governance_api: dict = self.get_score_api(GOVERNANCE_SCORE_ADDRESS) self.assertNotEqual(original_governance_api, updated_governance_api) def test_exists_score_revision3_abnormal_scores_audit(self): self._setUp_audit() # set revision to 3 tx_results: List['TransactionResult'] = self.update_governance("0_0_4") self.accept_score(tx_results[0].tx_hash) self.set_revision(3) sample_score_params = {"value": hex(1000)} # deploy SCORE(not python) tx_results: List['TransactionResult'] = self._deploy_score(score_root="sample_deploy_scores", score_name="install/test_score_no_python", from_=self._accounts[0], to_=SYSTEM_SCORE_ADDRESS, deploy_params=sample_score_params) self.accept_score(tx_hash=tx_results[0].tx_hash, expected_status=False) # deploy SCORE(has no external function) tx_results: List['TransactionResult'] = self._deploy_score(score_root="sample_deploy_scores", score_name="install/test_score_no_external_func", from_=self._accounts[0], to_=SYSTEM_SCORE_ADDRESS, deploy_params=sample_score_params) self.accept_score(tx_hash=tx_results[0].tx_hash, expected_status=False) # deploy SCORE(no scorebase) tx_results: List['TransactionResult'] = self._deploy_score(score_root="sample_deploy_scores", score_name="install/test_score_no_scorebase", from_=self._accounts[0], to_=SYSTEM_SCORE_ADDRESS, deploy_params=sample_score_params) self.accept_score(tx_hash=tx_results[0].tx_hash, expected_status=False) # deploy SCORE(on install error) tx_results: List['TransactionResult'] = self._deploy_score(score_root="sample_deploy_scores", score_name="install/test_on_install_error", from_=self._accounts[0], to_=SYSTEM_SCORE_ADDRESS, deploy_params=sample_score_params) self.accept_score(tx_hash=tx_results[0].tx_hash, expected_status=False) # deploy SCORE(different encoding) tx_results: List['TransactionResult'] = self._deploy_score(score_root="sample_deploy_scores", score_name="install/test_score_with_korean_comment", from_=self._accounts[0], to_=SYSTEM_SCORE_ADDRESS, deploy_params=sample_score_params) self.accept_score(tx_hash=tx_results[0].tx_hash, expected_status=False)
class IconServiceSyncer(object): _TAG = "SYNC" def __init__(self): self._block_reader = BlockDatabaseReader() self._engine = IconServiceEngine() def open(self, config_path: str, fee: bool = True, audit: bool = True, deployer_whitelist: bool = False, score_package_validator: bool = False, builtin_score_owner: str = ''): conf = IconConfig("", default_icon_config) if config_path != "": conf.load(config_path) conf.update_conf({ "builtinScoreOwner": builtin_score_owner, "service": { "fee": fee, "audit": audit, "scorePackageValidator": score_package_validator, "deployerWhiteList": deployer_whitelist } }) Logger.load_config(conf) self._engine.open(conf) def run(self, *args, **kwargs) -> int: Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}") loop = asyncio.get_event_loop() future = asyncio.Future() try: asyncio.ensure_future( self._wait_for_complete(future, *args, **kwargs)) loop.run_until_complete(future) finally: self._engine.close() loop.close() ret = future.result() Logger.debug(tag=self._TAG, msg=f"run() end: {ret}") return ret async def _wait_for_complete(self, result_future: asyncio.Future, *args, **kwargs): Logger.debug(tag=self._TAG, msg="_wait_for_complete() start") executor = ThreadPoolExecutor(max_workers=1) f = executor.submit(self._run, *args, **kwargs) future = asyncio.wrap_future(f) await future Logger.debug(tag=self._TAG, msg="_wait_for_complete() end1") result_future.set_result(future.result()) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end2") def _run(self, db_path: str, channel: str, start_height: int = 0, count: int = 99999999, stop_on_error: bool = True, no_commit: bool = False, backup_period: int = 0, write_precommit_data: bool = False) -> int: """Begin to synchronize IconServiceEngine with blocks from loopchain db :param db_path: loopchain db path :param channel: channel name used as a key to get commit_state in loopchain db :param start_height: start height to sync :param count: The number of blocks to sync :param stop_on_error: If error happens, stop syncing :param no_commit: Do not commit :param backup_period: state backup period in block :param write_precommit_data: :return: 0(success), otherwise(error) """ Logger.debug(tag=self._TAG, msg="_run() start") ret: int = 0 self._block_reader.open(db_path) print('block_height | commit_state | state_root_hash | tx_count') prev_block: Optional['Block'] = None for height in range(start_height, start_height + count): block_dict: dict = self._block_reader.get_block_by_block_height( height) if block_dict is None: print(f'last block: {height - 1}') break loopchain_block = LoopchainBlock.from_dict(block_dict) block: 'Block' = utils.create_block(loopchain_block) tx_requests: list = utils.create_transaction_requests( loopchain_block) if prev_block is not None: # print(f'prev_block({prev_block.hash.hex()}) == block({block.prev_hash.hex()})') if prev_block.hash != block.prev_hash: raise Exception() invoke_result = self._engine.invoke(block, tx_requests) tx_results, state_root_hash = invoke_result[0], invoke_result[1] commit_state: bytes = self._block_reader.get_commit_state( block_dict, channel, b'') # "commit_state" is the field name of state_root_hash in loopchain block print( f'{height} | {commit_state.hex()[:6]} | {state_root_hash.hex()[:6]} | {len(tx_requests)}' ) if write_precommit_data: self._print_precommit_data(block) try: if stop_on_error: if commit_state: if commit_state != state_root_hash: raise Exception() if height > 0 and not self._check_invoke_result( tx_results): raise Exception() except Exception as e: logging.exception(e) print(block_dict) self._print_precommit_data(block) ret: int = 1 break if not no_commit: if 'block' in inspect.signature( self._engine.commit).parameters: self._engine.commit(block) else: self._engine.commit(block.height, block.hash, None) self._backup_state_db(block, backup_period) prev_block = block self._block_reader.close() Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}") return ret def _check_invoke_result(self, tx_results: list): """Compare the transaction results from IconServiceEngine with the results stored in loopchain db If transaction result is not compatible to protocol v3, pass it :param tx_results: the transaction results that IconServiceEngine.invoke() returns :return: True(same) False(different) """ for tx_result in tx_results: tx_info_in_db: dict =\ self._block_reader.get_transaction_result_by_hash( tx_result.tx_hash.hex()) tx_result_in_db = tx_info_in_db['result'] # tx_v2 dose not have transaction result_v3 if 'status' not in tx_result_in_db: continue # information extracted from db status: int = int(tx_result_in_db['status'], 16) tx_hash: bytes = bytes.fromhex(tx_result_in_db['txHash']) step_used: int = int(tx_result_in_db['stepUsed'], 16) step_price: int = int(tx_result_in_db['stepPrice'], 16) event_logs: list = tx_result_in_db['eventLogs'] step: int = step_used * step_price if tx_hash != tx_result.tx_hash: print(f'tx_hash: {tx_hash.hex()} != {tx_result.tx_hash.hex()}') return False if status != tx_result.status: print(f'status: {status} != {tx_result.status}') return False if step_used != tx_result.step_used: print(f'step_used: {step_used} != {tx_result.step_used}') return False tx_result_step: int = tx_result.step_used * tx_result.step_price if step != tx_result_step: print(f'step: {step} != {tx_result_step}') return False if step_price != tx_result.step_price: print(f'step_price: {step_price} != {tx_result.step_price}') return False if not self._check_event_logs(event_logs, tx_result.event_logs): return False return True @staticmethod def _check_event_logs(event_logs_in_db: list, event_logs_in_tx_result: list): for event_log, _tx_result_event_log in zip(event_logs_in_db, event_logs_in_tx_result): tx_result_event_log: dict = _tx_result_event_log.to_dict() # convert Address to str if 'score_address' in tx_result_event_log: score_address: 'Address' = tx_result_event_log['score_address'] del tx_result_event_log['score_address'] tx_result_event_log['scoreAddress'] = str(score_address) # convert Address objects to str objects in 'indexes' indexed: list = tx_result_event_log['indexed'] for i in range(len(indexed)): value = indexed[i] indexed[i] = utils.object_to_str(value) data: list = tx_result_event_log['data'] for i in range(len(data)): value = data[i] data[i] = utils.object_to_str(value) if event_log != tx_result_event_log: print(f'{event_log} != {tx_result_event_log}') return False return True def _print_precommit_data(self, block: 'Block'): """Print the latest updated states stored in IconServiceEngine :return: """ precommit_data_manager: PrecommitDataManager =\ getattr(self._engine, '_precommit_data_manager') precommit_data: PrecommitData = precommit_data_manager.get(block.hash) block_batch: BlockBatch = precommit_data.block_batch state_root_hash: bytes = block_batch.digest() filename = f'{block.height}-precommit-data.txt' with open(filename, 'wt') as f: for i, key in enumerate(block_batch): value: bytes = block_batch[key] if value: line = f'{i}: {key.hex()} - {value.hex()}' else: line = f'{i}: {key.hex()} - None' print(line) f.write(f'{line}\n') f.write(f'state_root_hash: {state_root_hash.hex()}\n') @staticmethod def _backup_state_db(block: 'Block', backup_period: int): if backup_period <= 0: return if block.height == 0: return if block.height % backup_period == 0: print(f"----------- Backup statedb: {block.height} ------------") dirname: str = f"block-{block.height}" for basename in (".score", ".statedb"): try: shutil.copytree(basename, f"{dirname}/{basename}/") except FileExistsError: pass def close(self): pass
class TestIntegrateExistentScoresAudit(TestIntegrateBase): # override setUp method for making directory before begin tests. def setUp(self): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None self.config = IconConfig("", default_icon_config) self.config.load() self.config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin)}) self.config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) self.config.update_conf(self._make_init_config()) def _setUp_audit(self): self.config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: True, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False}}) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self.config) self._genesis_invoke() self.token_initial_params = {"init_supply": hex(1000), "decimal": "0x12"} def _update_governance(self, governance_path): tx = self._make_deploy_tx("test_builtin", governance_path, self._admin, GOVERNANCE_SCORE_ADDRESS) prev_block, tx_results = self._make_and_req_block([tx]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) return tx['params']['txHash'] def _set_revision(self, revision): set_revision_tx = self._make_score_call_tx(self._admin, GOVERNANCE_SCORE_ADDRESS, 'setRevision', {"code": hex(revision), "name": f"1.1.{revision}"}) prev_block, tx_results = self._make_and_req_block([set_revision_tx]) self._write_precommit_state(prev_block) def _make_directories_in_builtin_score_path(self): for score_name, address in BUILTIN_SCORE_ADDRESS_MAPPER.items(): os.makedirs(os.path.join(self._score_root_path, f'01{address[2:]}', f"0x{'0'*64}"), exist_ok=True) def _make_directory_using_address_and_hash(self, score_address: 'Address', tx_hash: bytes): tx_str = f"0x{bytes.hex(tx_hash)}" os.makedirs(os.path.join(self._score_root_path, f"01{str(score_address)[2:]}", tx_str)) def _deploy_score(self, path: str, score_name: str, owner: 'Address', to: 'Address', deploy_params: dict): timestamp = int(time.time()*10**6) tx1 = self._make_deploy_tx(path, score_name, owner, to, deploy_params, timestamp) score_address = generate_score_address(owner, timestamp, nonce=0) tx_hash = tx1['params']['txHash'] self._make_directory_using_address_and_hash(score_address, tx_hash) return tx1 def _accept_score(self, tx_hash: bytes): tx = self._make_score_call_tx(self._admin, GOVERNANCE_SCORE_ADDRESS, 'acceptScore', params={"txHash": f'0x{bytes.hex(tx_hash)}'}) prev_block, tx_results = self._make_and_req_block([tx]) self._write_precommit_state(prev_block) return tx_results def test_existent_builtin_score_audit(self): self._make_directories_in_builtin_score_path() self._setUp_audit() # original SCORE api query_request = { "address": GOVERNANCE_SCORE_ADDRESS, } original_governance_api = self._query(query_request, 'icx_getScoreApi') # update governance SCORE(revision2) tx = self._update_governance('0_0_4') self._accept_score(tx) # updated SCORE api updated_governance_api = self._query(query_request, "icx_getScoreApi") self.assertNotEqual(original_governance_api, updated_governance_api) # test when revision <= 2 def test_existent_score_audit(self): self._setUp_audit() # deploy SCORE tx1 = self._deploy_score('test_deploy_scores/install', 'sample_token', self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) accept_results = self._accept_score(tx1['params']['txHash']) self.assertTrue("is a directory. Check " in accept_results[0].failure.message) self.assertEqual(accept_results[0].status, int(False)) # update governance SCORE(revision2) self._update_governance('0_0_4') # deploy tx2 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx2]) self._write_precommit_state(prev_block) accept_results = self._accept_score(tx2['params']['txHash']) self.assertTrue("is a directory. Check " in accept_results[0].failure.message) self.assertEqual(accept_results[0].status, int(False)) # test when revision > 2 def test_existent_score_revision4_audit(self): self._setUp_audit() # set revision to 4 update_governance_tx = self._update_governance('0_0_4') self._accept_score(update_governance_tx) self._set_revision(3) # deploy SCORE tx1 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) score_addr1 = tx_results[0].score_address # accept SCORE self._accept_score(tx1['params']['txHash']) # balance_of test(1000) query_request = { "from": self._admin, "to": score_addr1, "dataType": "call", "data": { "method": "balance_of", "params": { "addr_from": str(self._addr_array[0]) } } } response = self._query(query_request) self.assertEqual(response, 1000 * 10 ** 18) # deploy SCORE(update) tx2 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], score_addr1, {"update_supply": hex(3000), "decimal": "0x12"}) prev_block, tx_results = self._make_and_req_block([tx2]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) # accept SCORE self._accept_score(tx2['params']['txHash']) # balance_of test(3000) query_request['data']['params']['addr_from'] = str(self._addr_array[0]) response = self._query(query_request) self.assertEqual(response, 3000 * 10 ** 18) def test_rolling_update_deploy_audit(self): # case revision 0 self._setUp_audit() # deploy (revision0 must be fail) tx1 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) # accept SCORE accept_result = self._accept_score(tx1['params']['txHash']) self.assertTrue("is a directory." in accept_result[0].failure.message) self.assertEqual(accept_result[0].status, int(False)) # update governance SCORE(revision 2) tx2 = self._update_governance('0_0_4') self._accept_score(tx2) # deploy (revision2 must be success) tx2 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx2]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx2['params']['txHash']) self.assertEqual(accept_result[0].status, int(False)) # set revision to 4 self._set_revision(3) # deploy (revision4 must be success) tx4 = self._deploy_score("test_deploy_scores/install", "sample_token", self._addr_array[0], ZERO_SCORE_ADDRESS, self.token_initial_params) prev_block, tx_results = self._make_and_req_block([tx4]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx4['params']['txHash']) self.assertEqual(accept_result[0].status, int(True)) def test_exists_score_revision3_unnormal_scores_aduit(self): self._setUp_audit() # set revision to 3 update_governance_tx = self._update_governance('0_0_4') self._accept_score(update_governance_tx) self._set_revision(3) sample_score_params = {"value": hex(1000)} # deploy SCORE(not python) tx1 = self._deploy_score("test_deploy_scores/install", "test_score_no_python", self._addr_array[0], ZERO_SCORE_ADDRESS, sample_score_params) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx1['params']['txHash']) self.assertEqual(accept_result[0].status, int(False)) # deploy SCORE(has no external function) tx2 = self._deploy_score("test_deploy_scores/install", "test_score_no_external_func", self._addr_array[0], ZERO_SCORE_ADDRESS, sample_score_params) prev_block, tx_results = self._make_and_req_block([tx2]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx2['params']['txHash']) self.assertEqual(accept_result[0].status, int(False)) # deploy SCORE(no scorebase) tx3 = self._deploy_score("test_deploy_scores/install", "test_score_no_scorebase", self._addr_array[0], ZERO_SCORE_ADDRESS, sample_score_params) prev_block, tx_results = self._make_and_req_block([tx3]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx3['params']['txHash']) self.assertEqual(accept_result[0].status, int(False)) # deploy SCORE(on install error) tx4 = self._deploy_score("test_deploy_scores/install", "test_on_install_error", self._addr_array[0], ZERO_SCORE_ADDRESS, sample_score_params) prev_block, tx_results = self._make_and_req_block([tx4]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx4['params']['txHash']) self.assertEqual(accept_result[0].status, int(False)) # deploy SCORE(different encoding) tx5 = self._deploy_score("test_deploy_scores/install", "test_score_with_korean_comment", self._addr_array[0], ZERO_SCORE_ADDRESS, sample_score_params) prev_block, tx_results = self._make_and_req_block([tx5]) self._write_precommit_state(prev_block) accept_result = self._accept_score(tx5['params']['txHash']) self.assertEqual(accept_result[0].status, int(False))
class IconServiceSyncer(object): _TAG = "SYNC" def __init__(self): self._block_reader = BinBlockDatabaseReader() self._engine = IconServiceEngine() self._timer = Timer() def open( self, config_path: str, fee: bool = True, audit: bool = True, deployer_whitelist: bool = False, score_package_validator: bool = False, builtin_score_owner: str = "", ): conf = IconConfig("", default_icon_config) if config_path != "": conf.load(config_path) conf.update_conf({ "builtinScoreOwner": builtin_score_owner, "service": { "fee": fee, "audit": audit, "scorePackageValidator": score_package_validator, "deployerWhiteList": deployer_whitelist, }, }) Logger.load_config(conf) self._engine.open(conf) def run(self, *args, **kwargs) -> int: Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}") loop = asyncio.get_event_loop() future = asyncio.Future() try: asyncio.ensure_future( self._wait_for_complete(future, *args, **kwargs)) loop.run_until_complete(future) finally: for task in asyncio.Task.all_tasks(): task.cancel() loop.close() ret = future.result() Logger.debug(tag=self._TAG, msg=f"run() end: {ret}") return ret async def _wait_for_complete(self, result_future: asyncio.Future, *args, **kwargs): Logger.debug(tag=self._TAG, msg="_wait_for_complete() start") # Wait for rc to be ready future = self._engine.get_ready_future() await future with ThreadPoolExecutor(max_workers=1) as executor: # Call IconServiceEngine.hello() f = executor.submit(self._hello) future = asyncio.wrap_future(f) await future # Start to sync blocks f = executor.submit(self._run, *args, **kwargs) f = asyncio.wrap_future(f) for fut in asyncio.as_completed([f]): try: ret = await fut except Exception as exc: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) result_future.set_exception(exc) else: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end1") result_future.set_result(ret) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end2") def _hello(self): self._engine.hello() def _run( self, db_path: str, channel: str, start_height: int = 0, count: int = 99_999_999, stop_on_error: bool = True, no_commit: bool = False, backup_period: int = 0, write_precommit_data: bool = False, print_block_height: int = 1, iiss_db_backup_path: Optional[str] = None, ) -> int: """Begin to synchronize IconServiceEngine with blocks from loopchain db :param db_path: loopchain db path :param channel: channel name used as a key to get commit_state in loopchain db :param start_height: start height to sync :param count: The number of blocks to sync :param stop_on_error: If error happens, stop syncing :param no_commit: Do not commit :param backup_period: state backup period in block :param write_precommit_data: :param print_block_height: print every this block height :return: 0(success), otherwise(error) """ Logger.debug(tag=self._TAG, msg="_run() start") word_detector = WordDetector( filename="iconservice.log", block_word=r"CALCULATE\(", release_word=r"CALCULATE_DONE\(", ) ret: int = 0 self._block_reader.open(db_path) print("block_height | commit_state | state_root_hash | tx_count") prev_bin_block: Optional["BinBlock"] = None prev_block: Optional["Block"] = None main_preps: Optional['NodeContainer'] = None next_main_preps: Optional["NodeContainer"] = None if start_height > 0: prev_bin_block: Optional[ BinBlock] = self._block_reader.get_block_by_height( start_height - 1) # init main_preps preps: list = self._block_reader.load_main_preps( reps_hash=prev_bin_block.reps_hash) main_preps: Optional['NodeContainer'] = NodeContainer.from_list( preps=preps) # when sync from the first block of term, have to initialize next_main_preps here # in that case, invoke_result[3] will be None on first block and can not update next_main_preps cur_bin_block: Optional[ BinBlock] = self._block_reader.get_block_by_height( start_height) block: "Block" = create_iconservice_block(cur_bin_block) if self._check_calculation_block(block): preps: list = self._block_reader.load_main_preps( reps_hash=cur_bin_block.reps_hash) next_main_preps: Optional[ 'NodeContainer'] = NodeContainer.from_list(preps=preps) end_height = start_height + count - 1 self._timer.start() for height in range(start_height, start_height + count): bin_block = self._block_reader.get_block_by_height(height) if bin_block is None: print(f"last block: {height - 1}") break block: "Block" = create_iconservice_block(bin_block) tx_requests: List[Dict[str, Any]] = create_transaction_requests( bin_block.transactions) prev_block_generator: Optional[ "Address"] = prev_bin_block.leader if prev_bin_block else None prev_block_validators: Optional[ List["Address"]] = create_block_validators( bin_block.prev_votes, prev_block_generator) prev_block_votes: Optional[List[ Tuple["Address", int]]] = create_prev_block_votes(bin_block.prev_votes, prev_block_generator, main_preps) Logger.info(tag=self._TAG, msg=f"prev_block_generator={prev_block_generator}") Logger.info(tag=self._TAG, msg=f"prev_block_validators={prev_block_validators}") Logger.info(tag=self._TAG, msg=f"prev_block_votes={prev_block_votes}") if prev_block is not None and prev_block.hash != block.prev_hash: raise Exception(f"Invalid prev_block_hash: height={height}") invoke_result = self._engine.invoke( block, tx_requests, prev_block_generator, prev_block_validators, prev_block_votes, ) tx_results, state_root_hash = invoke_result[0], invoke_result[1] main_preps_as_dict: Optional[Dict] = invoke_result[3] commit_state: bytes = bin_block.state_hash # "commit_state" is the field name of state_root_hash in loopchain block if (height - start_height) % print_block_height == 0: self._print_status(height, start_height, count, commit_state, state_root_hash, len(tx_requests)) if write_precommit_data: self._print_precommit_data(block) try: if stop_on_error: if commit_state: if commit_state != state_root_hash: raise Exception("state_root_hash mismatch") if height > 0 and not self._check_invoke_result( tx_results): raise Exception("tx_result mismatch") except Exception as e: logging.exception(e) self._print_precommit_data(block) ret: int = 1 break is_calculation_block = self._check_calculation_block(block) if is_calculation_block: word_detector.start() time.sleep(0.5) if iiss_db_backup_path is not None: self._backup_iiss_db(iiss_db_backup_path, block.height) # If no_commit is set to True, the config only affects to the last block to commit if not no_commit or height < end_height: while word_detector.get_hold(): time.sleep(0.5) # Call IconServiceEngine.commit() with a block self._commit(block) while word_detector.get_hold(): time.sleep(0.5) # Prepare the next iteration self._backup_state_db(block, backup_period) prev_block: 'Block' = block prev_bin_block: 'BinBlock' = bin_block if next_main_preps: main_preps = next_main_preps next_main_preps = None if main_preps_as_dict is not None: next_main_preps = NodeContainer.from_dict(main_preps_as_dict) self._block_reader.close() word_detector.stop() Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}") return ret
class IconIntegrateTestBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.testscore' cls._state_db_root_path = '.teststatedb' cls._icx_factor = 10 ** 18 cls._genesis: 'KeyWallet' = KeyWallet.create() cls._fee_treasury: 'KeyWallet' = KeyWallet.create() cls._test1: 'KeyWallet' = KeyWallet.load(bytes.fromhex(TEST1_PRIVATE_KEY)) cls._wallet_array = [KeyWallet.create() for _ in range(10)] def setUp(self): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: self._test1.get_address()}) config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke() def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} def _genesis_invoke(self) -> dict: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': 3, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [ { "name": "genesis", "address": Address.from_string(self._genesis.get_address()), "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": Address.from_string(self._fee_treasury.get_address()), "balance": 0 }, { "name": "_admin", "address": Address.from_string(self._test1.get_address()), "balance": 1_000_000 * self._icx_factor } ] }, }
class IconIntegrateTestBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.testscore' cls._state_db_root_path = '.teststatedb' cls._icx_factor = 10 ** 18 cls._genesis: 'KeyWallet' = KeyWallet.create() cls._fee_treasury: 'KeyWallet' = KeyWallet.create() cls._test1: 'KeyWallet' = KeyWallet.load(bytes.fromhex(TEST1_PRIVATE_KEY)) cls._wallet_array = [KeyWallet.create() for _ in range(10)] def setUp(self, genesis_accounts: List[Account] = None): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: self._test1.get_address()}) config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke(genesis_accounts) def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} @staticmethod def _append_list(tx: dict, genesis_accounts: List[Account]) -> None: """Appends additional genesis account list to genesisData :param genesis_accounts: additional genesis account list consisted of namedtuple named Account of which keys are name, address and balance :return: None """ for account_as_namedtuple in genesis_accounts: tx["genesisData"]['accounts'].append({ "name": account_as_namedtuple.name, "address": account_as_namedtuple.address, "balance": account_as_namedtuple.balance }) def _genesis_invoke(self, genesis_accounts: List[Account]) -> list: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': 3, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [ { "name": "genesis", "address": Address.from_string(self._genesis.get_address()), "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": Address.from_string(self._fee_treasury.get_address()), "balance": 0 }, { "name": "_admin", "address": Address.from_string(self._test1.get_address()), "balance": 1_000_000 * self._icx_factor } ] }, }
class TestIntegrateBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.score' cls._state_db_root_path = '.statedb' cls._iiss_db_root_path = '.iissdb' cls._test_sample_root = "samples" cls._signature = "VAia7YZ2Ji6igKWzjR2YsGa2m53nKPrfK7uXYW78QLE+ATehAVZPC40szvAiA6NEU5gCYB4c4qaQzqDh2ugcHgA=" cls._version = 3 cls._admin: 'EOAAccount' = cls.create_eoa_accounts(1)[0] cls._genesis: 'Address' = create_address() cls._fee_treasury: 'Address' = create_address() cls._accounts = cls.create_eoa_accounts(100) cls._tx_results: dict = {} def setUp(self): root_clear(self._score_root_path, self._state_db_root_path, self._iiss_db_root_path) self._block_height = -1 self._prev_block_hash = None config = IconConfig("", copy.deepcopy(default_icon_config)) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin.address)}) config.update_conf({ConfigKey.SERVICE: {ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False}}) config.update_conf({ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path}) config.update_conf(self._make_init_config()) self._config: 'IconConfig' = config self.icon_service_engine = IconServiceEngine() self._mock_ipc() self.icon_service_engine.open(config) self._genesis_invoke() def get_block_height(self) -> int: return self._block_height def mock_calculate(self, _path, _block_height): context: 'IconScoreContext' = IconScoreContext(IconScoreContextType.QUERY) end_block_height_of_calc: int = context.storage.iiss.get_end_block_height_of_calc(context) calc_period: int = context.storage.iiss.get_calc_period(context) response = CalculateDoneNotification(0, True, end_block_height_of_calc - calc_period, 0, b'mocked_response') self._calculate_done_callback(response) @classmethod def _mock_ipc(cls, mock_calculate: callable = mock_calculate): RewardCalcProxy.open = Mock() RewardCalcProxy.start = Mock() RewardCalcProxy.stop = Mock() RewardCalcProxy.close = Mock() RewardCalcProxy.get_version = Mock() RewardCalcProxy.calculate = mock_calculate RewardCalcProxy.claim_iscore = Mock() RewardCalcProxy.query_iscore = Mock() RewardCalcProxy.commit_block = Mock() RewardCalcProxy.commit_claim = Mock() RewardCalcProxy.query_calculate_result = Mock(return_value=(RCCalculateResult.SUCCESS, 0, 0, bytes())) def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path, self._iiss_db_root_path) def _make_init_config(self) -> dict: return {} def _genesis_invoke(self) -> tuple: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': self._version, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [ { "name": "genesis", "address": self._genesis, "balance": 0 }, { "name": "fee_treasury", "address": self._fee_treasury, "balance": 0 }, { "name": "_admin", "address": self._admin.address, "balance": icx_to_loop(TOTAL_SUPPLY) } ] }, } block_hash = create_block_hash() block = Block(self._block_height + 1, block_hash, timestamp_us, None, 0) invoke_response: tuple = self.icon_service_engine.invoke( block, [tx] ) self.icon_service_engine.commit(block.height, block.hash, None) self._block_height += 1 self._prev_block_hash = block_hash return invoke_response def get_tx_results(self, hash_list: List[bytes]): tx_results: List['TransactionResult'] = [] for tx_hash in hash_list: tx_results.append(self._tx_results[tx_hash]) return tx_results @classmethod def get_hash_list_from_tx_list(cls, tx_list: list) -> List[bytes]: hash_list: list = [] for tx in tx_list: hash_list.append(tx['params']['txHash']) return hash_list def add_tx_result(self, tx_results: List['TransactionResult']): for tx_result in tx_results: self._tx_results[tx_result.tx_hash] = tx_result def make_and_req_block(self, tx_list: list, block_height: int = None, prev_block_generator: Optional['Address'] = None, prev_block_validators: Optional[List['Address']] = None, prev_block_votes: Optional[List[Tuple['Address', int]]] = None, block_hash: bytes = None) \ -> Tuple['Block', List[bytes]]: if block_height is None: block_height: int = self._block_height + 1 if block_hash is None: block_hash = create_block_hash() timestamp_us = create_timestamp() block = Block(block_height, block_hash, timestamp_us, self._prev_block_hash, 0) context = IconScoreContext(IconScoreContextType.DIRECT) is_block_editable = False self.icon_service_engine._set_revision_to_context(context) if context.is_decentralized(): is_block_editable = True tx_results, state_root_hash, added_transactions, main_prep_as_dict = \ self.icon_service_engine.invoke(block=block, tx_requests=tx_list, prev_block_generator=prev_block_generator, prev_block_validators=prev_block_validators, prev_block_votes=prev_block_votes, is_block_editable=is_block_editable) self.add_tx_result(tx_results) return block, self.get_hash_list_from_tx_list(tx_list) def debug_make_and_req_block(self, tx_list: list, prev_block_generator: Optional['Address'] = None, prev_block_validators: Optional[List['Address']] = None, prev_block_votes: Optional[List[Tuple['Address', int]]] = None, block: 'Block' = None) -> tuple: # Prevent a base transaction from being added to the original tx_list tx_list = copy.copy(tx_list) if block is None: block_height: int = self._block_height + 1 block_hash = create_block_hash() timestamp_us = create_timestamp() block = Block(block_height, block_hash, timestamp_us, self._prev_block_hash, 0) context = IconScoreContext(IconScoreContextType.DIRECT) is_block_editable = False self.icon_service_engine._set_revision_to_context(context) if context.is_decentralized(): is_block_editable = True tx_results, state_root_hash, added_transactions, main_prep_as_dict = \ self.icon_service_engine.invoke(block=block, tx_requests=tx_list, prev_block_generator=prev_block_generator, prev_block_validators=prev_block_validators, prev_block_votes=prev_block_votes, is_block_editable=is_block_editable) return block, tx_results, state_root_hash, added_transactions, main_prep_as_dict def _make_and_req_block_for_issue_test(self, tx_list: list, block_height: int = None, prev_block_generator: Optional['Address'] = None, prev_block_validators: Optional[List['Address']] = None, prev_block_votes: Optional[List[Tuple['Address', int]]] = None, is_block_editable=False, cumulative_fee: int = 0) -> Tuple['Block', List[bytes]]: if block_height is None: block_height: int = self._block_height + 1 block_hash = create_block_hash() timestamp_us = create_timestamp() block = Block(block_height, block_hash, timestamp_us, self._prev_block_hash, cumulative_fee) tx_results, _, added_transactions, main_prep_as_dict = \ self.icon_service_engine.invoke(block=block, tx_requests=tx_list, prev_block_generator=prev_block_generator, prev_block_validators=prev_block_validators, prev_block_votes=prev_block_votes, is_block_editable=is_block_editable) self.add_tx_result(tx_results) return block, self.get_hash_list_from_tx_list(tx_list) def _write_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.commit(block.height, block.hash, None) self._block_height += 1 assert block.height == self._block_height self._prev_block_hash = block.hash def _remove_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.rollback(block.height, block.hash) def _query(self, request: dict, method: str = 'icx_call') -> Any: response = self.icon_service_engine.query(method, request) return response def inner_call(self, request: dict) -> Any: response = self.icon_service_engine.inner_call(request) return response def _create_invalid_block(self, block_height: int = None) -> 'Block': if block_height is None: block_height: int = self._block_height block_hash = create_block_hash() timestamp_us = create_timestamp() return Block(block_height, block_hash, timestamp_us, self._prev_block_hash, 0) @classmethod def _convert_address_from_address_type(cls, from_: Union[ 'EOAAccount', 'Address', 'MalformedAddress', None] ) -> Union['Address', 'MalformedAddress', None]: if isinstance(from_, EOAAccount): return from_.address elif isinstance(from_, (Address, MalformedAddress)): return from_ else: return None # ====== API ===== # def process_confirm_block_tx(self, tx_list: list, expected_status: bool = True, prev_block_generator: Optional['Address'] = None, prev_block_validators: Optional[List['Address']] = None, prev_block_votes: Optional[List[Tuple['Address', int]]] = None, block_height: int = None) -> List['TransactionResult']: prev_block, hash_list = self.make_and_req_block(tx_list, block_height, prev_block_generator, prev_block_validators, prev_block_votes) self._write_precommit_state(prev_block) tx_results: List['TransactionResult'] = self.get_tx_results(hash_list) for tx_result in tx_results: self.assertEqual(int(expected_status), tx_result.status) return tx_results def process_confirm_block(self, tx_list: list, prev_block_generator: Optional['Address'] = None, prev_block_validators: Optional[List['Address']] = None, prev_block_votes: Optional[List[Tuple['Address', int]]] = None, block_height: int = None) -> List['TransactionResult']: prev_block, hash_list = self.make_and_req_block(tx_list, block_height, prev_block_generator, prev_block_validators, prev_block_votes) self._write_precommit_state(prev_block) tx_results: List['TransactionResult'] = self.get_tx_results(hash_list) return tx_results def create_deploy_score_tx(self, score_root: str, score_name: str, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address'], deploy_params: dict = None, timestamp_us: int = None, data: bytes = None, is_sys: bool = False, pre_validation_enabled: bool = True, step_limit: int = DEFAULT_DEPLOY_STEP_LIMIT) -> dict: addr_from: Optional['Address'] = self._convert_address_from_address_type(from_) addr_to: 'Address' = self._convert_address_from_address_type(to_) if deploy_params is None: deploy_params = {} score_path = get_score_path(score_root, score_name) if is_sys: deploy_data = {'contentType': 'application/tbears', 'content': score_path, 'params': deploy_params} else: if data is None: mz = InMemoryZip() mz.zip_in_memory(score_path) data = f'0x{mz.data.hex()}' else: data = f'0x{bytes.hex(data)}' deploy_data = {'contentType': 'application/zip', 'content': data, 'params': deploy_params} if timestamp_us is None: timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "stepLimit": step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "deploy", "data": deploy_data } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = { 'method': method, 'params': request_params } if pre_validation_enabled: self.icon_service_engine.validate_transaction(tx) return tx def create_score_call_tx(self, from_: Union['EOAAccount', 'Address', None], to_: 'Address', func_name: str, params: Optional[dict] = None, value: int = 0, pre_validation_enabled: bool = True, step_limit: int = DEFAULT_BIG_STEP_LIMIT) -> dict: from_: Optional['Address'] = self._convert_address_from_address_type(from_) if params is None: params: dict = {} timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": from_, "to": to_, "value": value, "stepLimit": step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "call", "data": { "method": func_name, "params": params } } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = { 'method': method, 'params': request_params } if pre_validation_enabled: self.icon_service_engine.validate_transaction(tx) return tx def create_transfer_icx_tx(self, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address', 'MalformedAddress'], value: int, disable_pre_validate: bool = False, support_v2: bool = False, step_limit: int = DEFAULT_STEP_LIMIT) -> dict: addr_from: Optional['Address'] = self._convert_address_from_address_type(from_) addr_to: Optional['Address', 'MalformedAddress'] = self._convert_address_from_address_type(to_) timestamp_us = create_timestamp() nonce = 0 request_params = { "from": addr_from, "to": addr_to, "value": value, "stepLimit": step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature } if support_v2: request_params["fee"] = 10 ** 16 else: request_params["version"] = self._version method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = { 'method': method, 'params': request_params } if not disable_pre_validate: self.icon_service_engine.validate_transaction(tx) return tx def create_message_tx(self, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address', 'MalformedAddress'], data: bytes = None, value: int = 0) -> dict: addr_from: Optional['Address'] = self._convert_address_from_address_type(from_) addr_to: Optional['Address', 'MalformedAddress'] = self._convert_address_from_address_type(to_) timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "value": value, "stepLimit": DEFAULT_BIG_STEP_LIMIT, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "message", "data": '0x' + data.hex(), } method = 'icx_sendTransaction' # Inserts txHash into request params request_params['txHash'] = create_tx_hash() tx = { 'method': method, 'params': request_params } self.icon_service_engine.validate_transaction(tx) return tx def create_deposit_tx(self, from_: Union['EOAAccount', 'Address', None], to_: 'Address', action: str, params: dict, value: int = 0, pre_validation_enabled: bool = True, step_limit: int = DEFAULT_BIG_STEP_LIMIT) -> dict: addr_from: Optional['Address'] = self._convert_address_from_address_type(from_) addr_to: 'Address' = self._convert_address_from_address_type(to_) timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "value": value, "stepLimit": step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "deposit", "data": { "action": action, } } for k, v in params.items(): request_params["data"][k] = v method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = { 'method': method, 'params': request_params } if pre_validation_enabled: self.icon_service_engine.validate_transaction(tx) return tx def create_register_proposal_tx(self, from_: 'Address', title: str, description: str, type_: int, value: Union[str, int, 'Address'], step_limit: int = DEFAULT_BIG_STEP_LIMIT) -> dict: text = '{"address":"%s"}' % value json_data: bytes = text.encode("utf-8") method = "registerProposal" score_params = { "title": title, "description": description, "type": hex(type_), "value": bytes_to_hex(json_data) } return self.create_score_call_tx(from_=from_, to_=GOVERNANCE_SCORE_ADDRESS, func_name=method, params=score_params, step_limit=step_limit) def create_vote_proposal_tx(self, from_: 'Address', id_: bytes, vote: bool, step_limit=DEFAULT_BIG_STEP_LIMIT) -> dict: method = "voteProposal" score_params = { "id": bytes_to_hex(id_, "0x"), "vote": hex(vote) } return self.create_score_call_tx(from_=from_, to_=GOVERNANCE_SCORE_ADDRESS, func_name=method, params=score_params, step_limit=step_limit) @staticmethod def _convert_tx_for_estimating_step_from_origin_tx(tx: dict): tx = copy.deepcopy(tx) tx["method"] = "debug_estimateStep" del tx["params"]["nonce"] del tx["params"]["stepLimit"] del tx["params"]["timestamp"] del tx["params"]["txHash"] del tx["params"]["signature"] return tx # ===== wrapping API ===== # def estimate_step(self, tx: dict): converted_tx = self._convert_tx_for_estimating_step_from_origin_tx(tx) return self.icon_service_engine.estimate_step(request=converted_tx) def update_governance(self, version: str = "latest_version", expected_status: bool = True) -> List['TransactionResult']: tx = self.create_deploy_score_tx("sample_builtin", f"{version}/governance", self._admin, GOVERNANCE_SCORE_ADDRESS) return self.process_confirm_block_tx([tx], expected_status) def transfer_icx(self, from_: Union['EOAAccount', 'Address', None], to_: Union['EOAAccount', 'Address', 'MalformedAddress'], value: int, disable_pre_validate: bool = False, support_v2: bool = False, step_limit: int = DEFAULT_STEP_LIMIT, expected_status: bool = True) -> List['TransactionResult']: tx = self.create_transfer_icx_tx(from_=from_, to_=to_, value=value, disable_pre_validate=disable_pre_validate, support_v2=support_v2, step_limit=step_limit) return self.process_confirm_block_tx([tx], expected_status) def deploy_score(self, score_root: str, score_name: str, from_: Union['EOAAccount', 'Address', None], deploy_params: dict = None, step_limit: int = DEFAULT_DEPLOY_STEP_LIMIT, expected_status: bool = True, to_: Union['EOAAccount', 'Address'] = ZERO_SCORE_ADDRESS, data: bytes = None) -> List['TransactionResult']: tx = self.create_deploy_score_tx(score_root=score_root, score_name=score_name, from_=from_, to_=to_, deploy_params=deploy_params, step_limit=step_limit, data=data) return self.process_confirm_block_tx([tx], expected_status) def score_call(self, from_: Union['EOAAccount', 'Address', None], to_: 'Address', func_name: str, params: dict = None, value: int = 0, step_limit: int = DEFAULT_BIG_STEP_LIMIT, expected_status: bool = True) -> List['TransactionResult']: tx = self.create_score_call_tx(from_=from_, to_=to_, func_name=func_name, params=params, value=value, step_limit=step_limit) return self.process_confirm_block_tx([tx], expected_status) def set_revision(self, revision: int, expected_status: bool = True) -> List['TransactionResult']: return self.score_call(from_=self._admin, to_=GOVERNANCE_SCORE_ADDRESS, func_name="setRevision", params={"code": hex(revision), "name": f"1.1.{revision}"}, expected_status=expected_status) def accept_score(self, tx_hash: Union[bytes, str], warning_message: str = None, expected_status: bool = True) -> List['TransactionResult']: if isinstance(tx_hash, bytes): tx_hash_str = f'0x{bytes.hex(tx_hash)}' else: tx_hash_str = tx_hash params: dict = {"txHash": tx_hash_str} if warning_message is not None: params["warning"] = warning_message return self.score_call(from_=self._admin, to_=GOVERNANCE_SCORE_ADDRESS, func_name="acceptScore", params=params, expected_status=expected_status) def reject_score(self, tx_hash: Union[bytes, str], reason: str = "reason", expected_status: bool = True) -> List['TransactionResult']: if isinstance(tx_hash, bytes): tx_hash_str = f'0x{bytes.hex(tx_hash)}' else: tx_hash_str = tx_hash return self.score_call(from_=self._admin, to_=GOVERNANCE_SCORE_ADDRESS, func_name="rejectScore", params={"txHash": tx_hash_str, "reason": reason}, expected_status=expected_status) def deposit_icx(self, score_address: 'Address', amount: int, period: int, sender: Union['EOAAccount', 'Address', None] = None, expected_status: bool = True) -> List['TransactionResult']: if sender is None: sender = self._admin if FIXED_TERM: tx: dict = self.create_deposit_tx(from_=sender, to_=score_address, action="add", params={}, value=amount) else: tx: dict = self.create_deposit_tx(from_=sender, to_=score_address, action="add", params={"term": hex(period)}, value=amount) return self.process_confirm_block_tx([tx], expected_status) def withdraw_deposit(self, deposit_id: bytes, score_address: 'Address', sender: Union['EOAAccount', 'Address', None] = None, expected_status: bool = True) -> List['TransactionResult']: if sender is None: sender = self._admin tx: dict = self.create_deposit_tx(from_=sender, to_=score_address, action="withdraw", params={"id": f"0x{bytes.hex(deposit_id)}"}) return self.process_confirm_block_tx([tx], expected_status) def register_proposal(self, from_: 'Address', title: str, description: str, type_: int, value: Union[str, int, 'Address'], expected_status: bool = True) -> 'TransactionResult': tx: dict = self.create_register_proposal_tx(from_, title, description, type_, value) # 0: base transaction, 1: register proposal tx_results = self.process_confirm_block_tx([tx], expected_status) return tx_results[1] def vote_proposal(self, from_: 'Address', id_: bytes, vote: bool, expected_status: bool = True) -> List['TransactionResult']: tx: dict = self.create_vote_proposal_tx(from_, id_, vote) return self.process_confirm_block_tx([tx], expected_status) def get_balance(self, account: Union['EOAAccount', 'Address']) -> int: address: Optional['Address'] = self._convert_address_from_address_type(account) return self._query( request={ "address": address }, method="icx_getBalance" ) def get_total_supply(self) -> int: return self._query(request={}, method="icx_getTotalSupply") def get_score_api(self, address: 'Address'): return self._query( request={ "address": address }, method='icx_getScoreApi' ) def query_score(self, from_: Union['EOAAccount', 'Address', None], to_: 'Address', func_name: str, params: dict = None): query_request = { "version": self._version, "from": from_, "to": to_, "dataType": "call", "data": { "method": func_name, "params": {} if params is None else params } } return self._query(query_request) def get_score_status(self, to_: 'Address'): query_request = { "version": self._version, "from": self._admin, "to": GOVERNANCE_SCORE_ADDRESS, "dataType": "call", "data": { "method": "getScoreStatus", "params": {"address": str(to_)} } } return self._query(query_request) def get_step_price(self) -> int: query_request = { "version": self._version, "from": self._admin, "to": GOVERNANCE_SCORE_ADDRESS, "dataType": "call", "data": { "method": "getStepPrice", "params": {} } } return self._query(query_request) @staticmethod def create_eoa_accounts(count: int) -> List['EOAAccount']: accounts: list = [] wallets: List['KeyWallet'] = [KeyWallet.create() for _ in range(count)] for wallet in wallets: accounts.append(EOAAccount(wallet)) return accounts
class TestIntegrateBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.score' cls._state_db_root_path = '.statedb' cls._test_sample_root = "" cls._signature = "VAia7YZ2Ji6igKWzjR2YsGa2m53nKPrfK7uXYW78QLE+ATehAVZPC40szvAiA6NEU5gCYB4c4qaQzqDh2ugcHgA=" cls._version = 3 cls._step_limit = 1 * 10**9 cls._icx_factor = 10**18 cls._admin: 'Address' = create_address() cls._genesis: 'Address' = create_address() cls._fee_treasury: 'Address' = create_address() cls._owner1: 'Address' = create_address() cls._owner2: 'Address' = create_address() cls._owner3: 'Address' = create_address() cls._owner4: 'Address' = create_address() cls._owner5: 'Address' = create_address() cls._owner6: 'Address' = create_address() cls._addr_array = [create_address() for _ in range(10)] def setUp(self): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin)}) config.update_conf({ ConfigKey.SERVICE: { ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITELIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False } }) config.update_conf({ ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke() def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} def _genesis_invoke(self) -> dict: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': self._version, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [ { "name": "genesis", "address": self._genesis, "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": self._fee_treasury, "balance": 0 }, { "name": "owner1", "address": self._owner1, "balance": 100 * self._icx_factor }, { "name": "owner2", "address": self._owner2, "balance": 100 * self._icx_factor }, { "name": "owner3", "address": self._owner3, "balance": 100 * self._icx_factor }, ] }, } block_hash = create_block_hash() block = Block(self._block_height, block_hash, timestamp_us, None) invoke_response: dict = self.icon_service_engine.invoke(block, [tx]) self.icon_service_engine.commit(block) self._block_height += 1 self._prev_block_hash = block_hash return invoke_response #Todo: need to be refactoring def _deploy_multisig_wallet(self): tx1 = self._make_deploy_tx( "", "multisig_wallet", self._addr_array[0], ZERO_SCORE_ADDRESS, deploy_params={ "_walletOwners": str("%s,%s,%s" % (str(self._owner1), str(self._owner2), str(self._owner3))), "_required": "0x02" }) prev_block, tx_results = self._make_and_req_block([tx1]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) multisig_score_addr = tx_results[0].score_address # check wallet owners query_request = { "version": self._version, "from": self._admin, "to": multisig_score_addr, "dataType": "call", "data": { "method": "getWalletOwners", "params": { "_offset": "0", "_count": "10" } } } response = self._query(query_request) expected_owners = [ str(self._owner1), str(self._owner2), str(self._owner3) ] self.assertEqual(response, expected_owners) # check requirements query_request = { "version": self._version, "from": self._admin, "to": multisig_score_addr, "dataType": "call", "data": { "method": "getRequirements", "params": {} } } response = self._query(query_request) expected_requirements = 2 self.assertEqual(response, expected_requirements) return multisig_score_addr # Todo: need to be refactoring def _deploy_multisig_wallet_and_token_score(self, token_total_supply: int, token_owner: Address): tx1 = self._make_deploy_tx( "", "multisig_wallet", self._addr_array[0], ZERO_SCORE_ADDRESS, deploy_params={ "_walletOwners": str("%s,%s,%s" % (str(self._owner1), str(self._owner2), str(self._owner3))), "_required": "0x02" }) tx2 = self._make_deploy_tx("", "standard_token", token_owner, ZERO_SCORE_ADDRESS, deploy_params={ "initialSupply": str(hex(token_total_supply)), "decimals": str(hex(0)) }) prev_block, tx_results = self._make_and_req_block([tx1, tx2]) self._write_precommit_state(prev_block) self.assertEqual(tx_results[0].status, int(True)) self.assertEqual(tx_results[1].status, int(True)) multisig_score_addr = tx_results[0].score_address token_score_addr = tx_results[1].score_address # check wallet owners query_request = { "version": self._version, "from": self._admin, "to": multisig_score_addr, "dataType": "call", "data": { "method": "getWalletOwners", "params": { "_offset": "0", "_count": "10" } } } response = self._query(query_request) expected_owners = [ str(self._owner1), str(self._owner2), str(self._owner3) ] self.assertEqual(response, expected_owners) # check requirements query_request = { "version": self._version, "from": self._admin, "to": multisig_score_addr, "dataType": "call", "data": { "method": "getRequirements", "params": {} } } response = self._query(query_request) expected_requirements = 2 self.assertEqual(response, expected_requirements) # check token supply query_request = { "version": self._version, "from": self._admin, "to": token_score_addr, "dataType": "call", "data": { "method": "balanceOf", "params": { '_owner': str(token_owner) } } } response = self._query(query_request) self.assertEqual(response, token_total_supply) return multisig_score_addr, token_score_addr def _make_deploy_tx(self, score_root: str, score_name: str, addr_from: Union['Address', None], addr_to: 'Address', deploy_params: dict = None, timestamp_us: int = None, data: bytes = None, is_sys: bool = False): if deploy_params is None: deploy_params = {} score_path = get_score_path(score_root, score_name) if is_sys: deploy_data = { 'contentType': 'application/tbears', 'content': score_path, 'params': deploy_params } else: if data is None: mz = InMemoryZip() mz.zip_in_memory(score_path) data = f'0x{mz.data.hex()}' else: data = f'0x{bytes.hex(data)}' deploy_data = { 'contentType': 'application/zip', 'content': data, 'params': deploy_params } if timestamp_us is None: timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "deploy", "data": deploy_data } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} self.icon_service_engine.validate_transaction(tx) return tx def _make_score_call_tx(self, addr_from: Optional['Address'], addr_to: 'Address', method: str, params: dict, value: int = 0): timestamp_us = create_timestamp() nonce = 0 request_params = { "version": self._version, "from": addr_from, "to": addr_to, "value": value, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature, "dataType": "call", "data": { "method": method, "params": params } } method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} self.icon_service_engine.validate_transaction(tx) return tx def _make_icx_send_tx(self, addr_from: Optional['Address'], addr_to: Union['Address', 'MalformedAddress'], value: int, disable_pre_validate: bool = False, support_v2: bool = False): timestamp_us = create_timestamp() nonce = 0 request_params = { "from": addr_from, "to": addr_to, "value": value, "stepLimit": self._step_limit, "timestamp": timestamp_us, "nonce": nonce, "signature": self._signature } if support_v2: request_params["fee"] = 10**16 else: request_params["version"] = self._version method = 'icx_sendTransaction' # Insert txHash into request params request_params['txHash'] = create_tx_hash() tx = {'method': method, 'params': request_params} if not disable_pre_validate: self.icon_service_engine.validate_transaction(tx) return tx def _make_and_req_block(self, tx_list: list, block_height: int = None) -> tuple: if block_height is None: block_height: int = self._block_height block_hash = create_block_hash() timestamp_us = create_timestamp() block = Block(block_height, block_hash, timestamp_us, self._prev_block_hash) invoke_response, _ = self.icon_service_engine.invoke(block, tx_list) return block, invoke_response def _write_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.commit(block) self._block_height += 1 self._prev_block_hash = block.hash def _remove_precommit_state(self, block: 'Block') -> None: self.icon_service_engine.rollback(block) def _query(self, request: dict, method: str = 'icx_call') -> Any: response = self.icon_service_engine.query(method, request) return response def _create_invalid_block(self, block_height: int = None) -> 'Block': if block_height is None: block_height: int = self._block_height block_hash = create_block_hash() timestamp_us = create_timestamp() return Block(block_height, block_hash, timestamp_us, self._prev_block_hash)
class TestIntegrateBase(TestCase): @classmethod def setUpClass(cls): cls._score_root_path = '.score' cls._state_db_root_path = '.statedb' cls._test_sample_root = "test_samples" cls._signature = "VAia7YZ2Ji6igKWzjR2YsGa2m53nKPrfK7uXYW78QLE+ATehAVZPC40szvAiA6NEU5gCYB4c4qaQzqDh2ugcHgA=" cls._version = 3 cls._step_limit = 1 * 10**12 cls._icx_factor = 10**18 cls._admin: 'Address' = create_address() cls._genesis: 'Address' = create_address() cls._fee_treasury: 'Address' = create_address() cls._addr_array = [create_address() for _ in range(10)] def setUp(self): root_clear(self._score_root_path, self._state_db_root_path) self._block_height = 0 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf({ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin)}) config.update_conf({ ConfigKey.SERVICE: { ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: False, ConfigKey.SERVICE_DEPLOYER_WHITELIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False } }) config.update_conf({ ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) config.update_conf(self._make_init_config()) self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke() def tearDown(self): self.icon_service_engine.close() root_clear(self._score_root_path, self._state_db_root_path) def _make_init_config(self) -> dict: return {} def _genesis_invoke(self) -> dict: tx_hash = create_tx_hash() timestamp_us = create_timestamp() request_params = { 'txHash': tx_hash, 'version': self._version, 'timestamp': timestamp_us } tx = { 'method': 'icx_sendTransaction', 'params': request_params, 'genesisData': { "accounts": [{ "name": "genesis", "address": self._genesis, "balance": 100 * self._icx_factor }, { "name": "fee_treasury", "address": self._fee_treasury, "balance": 0 }, { "name": "_admin", "address": self._admin, "balance": 1_000_000 * self._icx_factor }] }, }
class IconScoreInnerTask(object): def __init__(self, conf: dict): self._conf = conf self._thread_flag = ENABLE_THREAD_FLAG self._icon_service_engine = IconServiceEngine() self._open() self._thread_pool = { THREAD_INVOKE: ThreadPoolExecutor(1), THREAD_STATUS: ThreadPoolExecutor(1), THREAD_QUERY: ThreadPoolExecutor(1), THREAD_ESTIMATE: ThreadPoolExecutor(1), THREAD_VALIDATE: ThreadPoolExecutor(1) } def _open(self): Logger.info(tag=_TAG, msg="_open() start") self._icon_service_engine.open(self._conf) Logger.info(tag=_TAG, msg="_open() end") def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool: return (self._thread_flag & flag) == flag def _check_icon_service_ready(self): if not self._icon_service_engine.is_reward_calculator_ready(): raise ServiceNotReadyException("Reward Calculator is not ready") @staticmethod def _log_exception(e: BaseException, tag: str) -> None: Logger.exception(str(e), tag) Logger.error(str(e), tag) @message_queue_task async def hello(self): Logger.info(tag=_TAG, msg='hello() start') ready_future = self._icon_service_engine.get_ready_future() await ready_future if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello) else: ret = self._hello() Logger.info(tag=_TAG, msg='hello() end') return ret def _hello(self): return self._icon_service_engine.hello() def cleanup(self): Logger.info(tag=_TAG, msg="cleanup() start") # shutdown thread pool executors for executor in self._thread_pool.values(): executor.shutdown() # close ICON Service if self._icon_service_engine: self._icon_service_engine.close() self._icon_service_engine = None Logger.info(tag=_TAG, msg="cleanup() end") @message_queue_task async def close(self): Logger.info(tag=_TAG, msg="close() stop event loop") self._close() @staticmethod def _close(): asyncio.get_event_loop().stop() @message_queue_task async def invoke(self, request: dict) -> dict: Logger.debug(tag=_TAG, msg=f'invoke() start') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret: dict = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._invoke, request) else: ret: dict = self._invoke(request) Logger.debug(tag=_TAG, msg=f'invoke() end') return ret def _invoke(self, request: dict): """Process transactions in a block :param request: :return: """ Logger.info(tag=_TAG, msg=f'INVOKE Request: {request}') try: params = TypeConverter.convert(request, ParamType.INVOKE) converted_block_params = params['block'] block = Block.from_dict(converted_block_params) Logger.info(tag=_TAG, msg=f'INVOKE: BH={block.height}') converted_tx_requests = params['transactions'] convert_tx_result_to_dict: bool = 'isBlockEditable' in params converted_is_block_editable = params.get('isBlockEditable', False) converted_prev_block_generator = params.get('prevBlockGenerator') converted_prev_block_validators = params.get('prevBlockValidators') converted_prev_votes = params.get('prevBlockVotes') tx_results, state_root_hash, added_transactions, next_preps, is_shutdown = \ self._icon_service_engine.invoke( block=block, tx_requests=converted_tx_requests, prev_block_generator=converted_prev_block_generator, prev_block_validators=converted_prev_block_validators, prev_block_votes=converted_prev_votes, is_block_editable=converted_is_block_editable ) if convert_tx_result_to_dict: convert_tx_results = [tx_result.to_dict(to_camel_case) for tx_result in tx_results] else: # old version convert_tx_results = {bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case) for tx_result in tx_results} results = { 'txResults': convert_tx_results, 'stateRootHash': bytes.hex(state_root_hash), 'addedTransactions': added_transactions } if next_preps: results["prep"] = next_preps if is_shutdown: results["is_shutdown"] = True response = MakeResponse.make_response(results) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except InvalidBaseTransactionException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) finally: if self._icon_service_engine: self._icon_service_engine.clear_context_stack() Logger.info(tag=_TAG, msg=f'INVOKE Response: {json.dumps(response, cls=BytesToHexJSONEncoder)}') return response @message_queue_task async def query(self, request: dict) -> dict: try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) return await self._get_query_response(request) async def _get_query_response(self, request: dict) -> dict: try: value = await self._execute_query(request) if isinstance(value, Address): value = str(value) response = MakeResponse.make_response(value) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._icon_service_engine.clear_context_stack() return response async def _execute_query(self, request: dict): method_name: str = request['method'] if method_name == RPCMethod.DEBUG_ESTIMATE_STEP: method: callable = self._estimate args = [request] else: method: callable = self._query args = [request, method_name] if self._is_thread_flag_on(EnableThreadFlag.QUERY): return await asyncio.get_event_loop(). \ run_in_executor(self._thread_pool[QUERY_THREAD_MAPPER[method_name]], method, *args) else: return method(*args) def _estimate(self, request: dict): converted_request = TypeConverter.convert(request, ParamType.INVOKE_TRANSACTION) return self._icon_service_engine.estimate_step(converted_request) def _query(self, request: dict, method: str): converted_request = TypeConverter.convert(request, ParamType.QUERY) return self._icon_service_engine.query(method, converted_request['params']) @message_queue_task async def call(self, request: dict): Logger.info(tag=_TAG, msg=f'call() start: {request}') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._call, request) else: ret = self._call(request) Logger.info(tag=_TAG, msg=f'call() end: {ret}') return ret def _call(self, request: dict): try: response = self._icon_service_engine.inner_call(request) if isinstance(response, Address): response = str(response) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) return response @message_queue_task async def write_precommit_state(self, request: dict): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._write_precommit_state, request) else: ret = self._write_precommit_state(request) return ret def _write_precommit_state(self, request: dict) -> dict: Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Request: {request}') try: converted_params = TypeConverter.convert(request, ParamType.WRITE_PRECOMMIT) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] instant_block_hash: bytes = converted_params[ConstantKeys.OLD_BLOCK_HASH] block_hash = converted_params[ConstantKeys.NEW_BLOCK_HASH] Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE: ' f'BH={block_height} ' f'instant_block_hash={bytes_to_hex(instant_block_hash)} ' f'block_hash={bytes_to_hex(block_hash)}') self._icon_service_engine.commit(block_height, instant_block_hash, block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Response: {response}') return response @message_queue_task async def remove_precommit_state(self, request: dict): Logger.info(tag=_TAG, msg=f'remove_precommit_state() start') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) """ Unused API """ return {} @message_queue_task async def rollback(self, request: dict): """Go back to the state of the given previous block :param request: :return: """ Logger.info(tag=_TAG, msg=f"rollback() start") try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() response = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._rollback, request) else: response = self._rollback(request) Logger.info(tag=_TAG, msg=f"rollback() end") return response def _rollback(self, request: dict) -> dict: Logger.info(tag=_TAG, msg=f"ROLLBACK Request: {request}") try: converted_params = TypeConverter.convert(request, ParamType.ROLLBACK) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH] Logger.info(tag=_TAG, msg=f"ROLLBACK: BH={block_height} block_hash={bytes_to_hex(block_hash)}") response: dict = self._icon_service_engine.rollback(block_height, block_hash) response = MakeResponse.make_response(response) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except BaseException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) Logger.info(tag=_TAG, msg=f"ROLLBACK Response: {response}") return response @message_queue_task async def validate_transaction(self, request: dict): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.VALIDATE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_VALIDATE], self._validate_transaction, request) else: return self._validate_transaction(request) def _validate_transaction(self, request: dict): try: Logger.info(tag=_TAG, msg=f'validate_transaction Request: {request}') converted_request = TypeConverter.convert(request, ParamType.VALIDATE_TRANSACTION) self._icon_service_engine.validate_transaction(converted_request, request) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def change_block_hash(self, _params): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) return ExceptionCode.OK @message_queue_task async def dos_guard(self, params: dict) -> dict: try: Logger.info(tag=_TAG, msg=f'dos_guard: params: {params}') self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) try: self._icon_service_engine.dos_guard.run(_from=params["from"]) response = MakeResponse.make_response(ExceptionCode.OK) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) return response
class TestIntegrateFeeSharing(TestIntegrateBase): def setUp(self): root_clear(self._score_root_path, self._state_db_root_path, self._iiss_db_root_path) self._block_height = -1 self._prev_block_hash = None config = IconConfig("", default_icon_config) config.load() config.update_conf( {ConfigKey.BUILTIN_SCORE_OWNER: str(self._admin.address)}) config.update_conf({ ConfigKey.SERVICE: { ConfigKey.SERVICE_AUDIT: False, ConfigKey.SERVICE_FEE: True, ConfigKey.SERVICE_DEPLOYER_WHITE_LIST: False, ConfigKey.SERVICE_SCORE_PACKAGE_VALIDATOR: False } }) config.update_conf({ ConfigKey.SCORE_ROOT_PATH: self._score_root_path, ConfigKey.STATE_DB_ROOT_PATH: self._state_db_root_path }) config.update_conf(self._make_init_config()) self._mock_ipc() self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(config) self._genesis_invoke() self.update_governance(version="governance_for_fee2") tx_results: List['TransactionResult'] = self.deploy_score( score_root="sample_deploy_scores", score_name="install/sample_score_fee_sharing", from_=self._admin, deploy_params={"value": hex(100)}) self.score_address = tx_results[0].score_address tx_results: List['TransactionResult'] = self.deploy_score( score_root="sample_deploy_scores", score_name="install/sample_score_fee_sharing_inter_call", from_=self._admin, deploy_params={ "value": hex(100), "score_address": str(self.score_address) }) self.score_address2 = tx_results[0].score_address def tearDown(self): super().tearDown() # noinspection PyDefaultArgument def _query_score_info(self, address: Address): query_request = { "version": self._version, "from": self._admin, "to": GOVERNANCE_SCORE_ADDRESS, "dataType": "call", "data": { "method": "getScoreStatus", "params": { "address": str(address) } } } response = self._query(query_request) return response def test_deposit_fee(self): before_balance: int = self.get_balance(self._admin) tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_fee = tx_results[0].step_price * tx_results[0].step_used deposit_id = tx_results[0].tx_hash score_info = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) self.assertIn( deposit_id, map(lambda d: d['id'], score_info['depositInfo']['deposits'])) after_balance: int = self.get_balance(self._admin) self.assertEqual(before_balance - MIN_DEPOSIT_AMOUNT - deposit_fee, after_balance) def test_deposit_fee_eventlog(self): # deploy same score for tx_results: List['TransactionResult'] = self.deploy_score( score_root="sample_deploy_scores", score_name="install/sample_score_fee_sharing", from_=self._admin, deploy_params={"value": hex(100)}) same_score_address = tx_results[0].score_address # set revision 4 self.set_revision(Revision.FOUR.value) # success case: before IISS_REV revision, should charge fee about event log tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) step_used_before_iiss_rev = tx_results[0].step_used # set revision 5 (IISS_REV) self.set_revision(Revision.IISS.value) tx_results: List['TransactionResult'] = self.deposit_icx( score_address=same_score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) step_used_after_iiss_rev = tx_results[0].step_used # event log count: 101 , event log step:100 event_log_fee = 101 * 100 self.assertEqual(step_used_before_iiss_rev, step_used_after_iiss_rev + event_log_fee) def test_deposit_fee_icx_range(self): self.deposit_icx(score_address=self.score_address, amount=MAX_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) self.deposit_icx(score_address=self.score_address, amount=MAX_DEPOSIT_AMOUNT + 1, period=MIN_DEPOSIT_TERM, expected_status=False) self.deposit_icx(score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) self.deposit_icx(score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT - 1, period=MIN_DEPOSIT_TERM, expected_status=False) def test_deposit_fee_term_range(self): self.deposit_icx(score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MAX_DEPOSIT_TERM) # self.deposit_icx(score_address=self.score_address, # amount=MIN_DEPOSIT_AMOUNT, # period=MAX_DEPOSIT_TERM + 1, # expected_status=False) self.deposit_icx(score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) # self.deposit_icx(score_address=self.score_address, # amount=MIN_DEPOSIT_AMOUNT, # period=MIN_DEPOSIT_TERM - 1, # expected_status=False) def test_sharing_fee_case_score_0(self): # deposit icx self.deposit_icx(score_address=self.score_address, amount=3 * MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) user_balance: int = self.get_balance(self._admin) score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] initial_available_deposit: int = deposit_info['availableDeposit'] tx_results: List['TransactionResult'] = self.score_call( from_=self._admin, to_=self.score_address, func_name="set_value", params={"value": hex(100)}) fee_used = tx_results[0].step_used * tx_results[0].step_price after_call_user_balance: int = self.get_balance(self._admin) score_info = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] after_call_available_deposit: int = deposit_info['availableDeposit'] self.assertEqual(user_balance - fee_used, after_call_user_balance) self.assertEqual(initial_available_deposit, after_call_available_deposit) self.assertFalse(tx_results[0].to_dict().get('detailed_step_used')) def test_sharing_fee_case_score_50(self): # deposit icx self.deposit_icx(score_address=self.score_address, amount=3 * MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) user_balance: int = self.get_balance(self._admin) score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] initial_available_deposit: int = deposit_info['availableDeposit'] initial_available_virtual_step: int = deposit_info[ 'availableVirtualStep'] proportion: int = 50 # invoke score method tx_results: List['TransactionResult'] = self.score_call( from_=self._admin, to_=self.score_address, func_name="set_value", params={ "value": hex(100), "proportion": hex(proportion) }) # check result score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] after_call_available_deposit: int = deposit_info['availableDeposit'] admin_addr: 'Address' = self._admin.address user_used_fee: int = tx_results[0].step_used_details[ admin_addr] * tx_results[0].step_price score_used_fee: int = tx_results[0].step_used_details[ self.score_address] * tx_results[0].step_price after_call_user_balance: int = self.get_balance(self._admin) remaining_step: int = score_used_fee - initial_available_virtual_step * STEP_PRICE remaining_step: int = 0 if remaining_step <= 0 else remaining_step self.assertEqual(initial_available_deposit - remaining_step, after_call_available_deposit) self.assertEqual(user_balance - user_used_fee, after_call_user_balance) self.assertEqual(score_used_fee, user_used_fee) def test_sharing_fee_case_score_100(self): # deposit icx self.deposit_icx(score_address=self.score_address, amount=3 * MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) user_balance: int = self.get_balance(self._admin) score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] initial_available_deposit: int = deposit_info['availableDeposit'] initial_available_virtual_step: int = deposit_info[ 'availableVirtualStep'] # invoke score method tx_results: List['TransactionResult'] = self.score_call( from_=self._admin, to_=self.score_address, func_name="set_value", params={ "value": hex(100), "proportion": hex(100) }) # check result score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] after_call_available_deposit: int = deposit_info['availableDeposit'] score_used_fee: int = tx_results[0].step_used_details[ self.score_address] * tx_results[0].step_price after_call_user_balance: int = self.get_balance(self._admin) remaining_step: int = score_used_fee - initial_available_virtual_step * STEP_PRICE remaining_step: int = 0 if remaining_step <= 0 else remaining_step self.assertEqual(initial_available_deposit - remaining_step, after_call_available_deposit) self.assertEqual(user_balance, after_call_user_balance) self.assertFalse(tx_results[0].step_used_details.get(self._admin)) @unittest.skip("Will take over 8 minutes") def test_score_call_after_deposit_expired(self): # deposit icx self.deposit_icx(score_address=self.score_address, amount=3 * MIN_DEPOSIT_AMOUNT, period=1) score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] initial_available_deposit: int = deposit_info['availableDeposit'] self.assertGreater(initial_available_deposit, 0) # increase block_height for i in range(MIN_DEPOSIT_TERM): self.transfer_icx(from_=self._admin, to_=self._accounts[0], value=10**8) # invoke score method with self.assertRaises(InvalidRequestException) as e: self.create_score_call_tx(from_=self._admin, to_=self.score_address, func_name="set_value", params={ "value": hex(100), "proportion": hex(100) }) self.assertEqual(e.exception.message, "Out of deposit balance") # check result score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] after_destroyed_available_deposit: int = deposit_info[ 'availableDeposit'] self.assertEqual(after_destroyed_available_deposit, 0) def test_deposit_unauthorized_account(self): # give icx to tester self.transfer_icx(from_=self._admin, to_=self._accounts[0], value=10000 * ICX_IN_LOOP) # unauthorized account deposit 5000icx in SCORE tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM, sender=self._accounts[0], expected_status=False) self.assertTrue(tx_results[0].failure) def test_deposit_nonexistent_score(self): # give icx to tester self.transfer_icx(from_=self._admin, to_=self._accounts[0], value=10000 * ICX_IN_LOOP) # deposit icx in nonexistent SCORE with self.assertRaises(InvalidRequestException) as e: self.deposit_icx(score_address=Address.from_prefix_and_int( AddressPrefix.CONTRACT, 3), amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) def test_get_score_info_without_deposit(self): """ Given : The SCORE is deployed. When : The SCORE does not have any deposit yet. Then : There is not no deposit list and all of values like sharing proportion, available virtual step and available deposit is 0. """ score_info = self._query_score_info(self.score_address) self.assertNotIn('depositInfo', score_info) def test_get_score_info_with_deposits(self): """ Given : The SCORE is deployed. When : The SCORE has one or two deposits. Then : Checks if values like sharing proportion, available virtual step and available deposit is correct. """ amount_deposit = 5000 * ICX_IN_LOOP virtual_step_issuance1 = 40_000_000_000 virtual_step_issuance2 = 80_000_000_000 # Creates a deposit with 5000 ICX tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=amount_deposit, period=MIN_DEPOSIT_TERM) deposit_id1 = tx_results[0].tx_hash score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] self.assertEqual(deposit_info["scoreAddress"], self.score_address) self.assertEqual(deposit_id1, deposit_info["deposits"][0]['id']) self.assertEqual(len(deposit_info["deposits"]), 1) self.assertEqual(deposit_info["availableVirtualStep"], virtual_step_issuance1) self.assertEqual(deposit_info["availableDeposit"], amount_deposit * 90 // 100) # Creates a more deposit with 5000 * 2 ICX tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=amount_deposit * 2, period=MIN_DEPOSIT_TERM) deposit_id2 = tx_results[0].tx_hash score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] self.assertEqual(deposit_info["scoreAddress"], self.score_address) self.assertEqual(deposit_id1, deposit_info["deposits"][0]['id']) self.assertEqual(deposit_id2, deposit_info["deposits"][1]['id']) self.assertEqual(len(deposit_info["deposits"]), 2) self.assertEqual(deposit_info["availableVirtualStep"], virtual_step_issuance1 + virtual_step_issuance2) sum_of_available_deposit: int = 0 for i in range(len(deposit_info["deposits"])): sum_of_available_deposit += deposit_info["deposits"][i][ 'depositAmount'] * 90 // 100 self.assertEqual(deposit_info["availableDeposit"], sum_of_available_deposit) def test_add_multiple_deposits(self): """ Given : The SCORE is deployed. When : The SCORE has multiple deposits. Then : Checks if SCORE has multiple deposits without any problem. """ amount_deposit = MIN_DEPOSIT_AMOUNT # Creates more deposit with 5000000 ICX for _ in range(99): self.deposit_icx(score_address=self.score_address, amount=amount_deposit, period=MIN_DEPOSIT_TERM) score_info = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] self.assertEqual(len(deposit_info["deposits"]), 99) self.assertEqual(deposit_info["availableDeposit"], (amount_deposit - amount_deposit * 10 // 100) * len(deposit_info['deposits'])) def test_get_deposit_by_valid_id(self): """ Given : The SCORE is deployed. When : Tries to get deposit info by valid id. Then : Returns deposit info correctly. """ tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_id: bytes = tx_results[0].tx_hash score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) self.assertIn( deposit_id, map(lambda d: d['id'], score_info['depositInfo']['deposits'])) def test_withdraw_deposit_after_deposit(self): """ Given : The SCORE is deployed and deposit once. When : Withdraws the deposit. Then : Amount of availableDeposit is 0. """ tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_id: bytes = tx_results[0].tx_hash score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) self.assertIn( deposit_id, map(lambda d: d['id'], score_info['depositInfo']['deposits'])) tx_results: List['TransactionResult'] = self.withdraw_deposit( deposit_id=deposit_id, score_address=self.score_address) self.assertTrue(tx_results[0].status) event_log = tx_results[0].event_logs[0] self.assertEqual('DepositWithdrawn(bytes,Address,int,int)', event_log.indexed[0]) self.assertEqual(event_log.data[0], MIN_DEPOSIT_AMOUNT) # withdraw amount self.assertEqual(event_log.data[1], 0) # penalty amount score_info: dict = self._query_score_info(self.score_address) self.assertNotIn('depositInfo', score_info) def test_withdraw_deposit_with_penalty(self): """ Given : The SCORE is deployed, deposit once and . When : Withdraws the deposit. Then : Amount of availableDeposit is 0. """ tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_id: bytes = tx_results[0].tx_hash score_info: dict = self._query_score_info(self.score_address) self.assertIn('depositInfo', score_info) self.assertIn( deposit_id, map(lambda d: d['id'], score_info['depositInfo']['deposits'])) # invoke score method to use virtual step self.score_call(from_=self._admin, to_=self.score_address, func_name="set_value", params={ "value": hex(100), "proportion": hex(100) }) tx_results: List['TransactionResult'] = self.withdraw_deposit( deposit_id=deposit_id, score_address=self.score_address) self.assertTrue(tx_results[0].status) event_log = tx_results[0].event_logs[0] self.assertEqual('DepositWithdrawn(bytes,Address,int,int)', event_log.indexed[0]) self.assertTrue( event_log.data[0] < MIN_DEPOSIT_AMOUNT) # withdraw amount self.assertTrue(event_log.data[1] > 0) # penalty amount score_info: dict = self._query_score_info(self.score_address) self.assertNotIn('depositInfo', score_info) def test_withdraw_deposit_by_not_owner(self): """ Given : The SCORE is deployed and deposit. When : Try to withdraw by not owner. Then : Return tx result with failure and status is 0. """ self.transfer_icx(from_=self._admin, to_=self._accounts[0], value=10000 * ICX_IN_LOOP) # deposit icx tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_id: bytes = tx_results[0].tx_hash # withdraw by not owner tx_results: List['TransactionResult'] = self.withdraw_deposit( deposit_id=deposit_id, score_address=self.score_address, sender=self._accounts[0], expected_status=False) self.assertEqual(tx_results[0].failure.message, "Invalid sender") def test_withdraw_deposit_again_after_already_withdraw_one(self): """ Given : The SCORE is deployed and deposit. Sets proportion. When : Withdraws twice from same deposit. Then : Return tx result with failure and status is 0. """ # deposit icx tx_results: List['TransactionResult'] = self.deposit_icx( score_address=self.score_address, amount=MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) deposit_id: bytes = tx_results[0].tx_hash # withdraw self.withdraw_deposit(deposit_id=deposit_id, score_address=self.score_address) score_info: dict = self._query_score_info(self.score_address) self.assertNotIn('depositInfo', score_info) # withdraw again tx_results: List['TransactionResult'] = self.withdraw_deposit( deposit_id=deposit_id, score_address=self.score_address, expected_status=False) self.assertEqual(tx_results[0].failure.message, "Deposit not found") def test_inter_call_fee_sharing_proportion100(self): # deposit icx self.deposit_icx(score_address=self.score_address2, amount=3 * MIN_DEPOSIT_AMOUNT, period=MIN_DEPOSIT_TERM) user_balance: int = self.get_balance(self._admin) score_info: dict = self._query_score_info(self.score_address2) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] initial_available_deposit: int = deposit_info['availableDeposit'] initial_available_virtual_step: int = deposit_info[ 'availableVirtualStep'] # invoke score method tx_results: List['TransactionResult'] = self.score_call( from_=self._admin, to_=self.score_address2, func_name="set_other_score_value", params={ "value": hex(100), "proportion": hex(100), "other_score_proportion": hex(0) }) # check result score_info: dict = self._query_score_info(self.score_address2) self.assertIn('depositInfo', score_info) deposit_info: dict = score_info['depositInfo'] after_call_available_deposit: int = deposit_info['availableDeposit'] score_used_fee: int = tx_results[0].step_used_details[ self.score_address2] * tx_results[0].step_price after_call_user_balance: int = self.get_balance(self._admin) remaining_step: int = score_used_fee - initial_available_virtual_step * STEP_PRICE remaining_step: int = 0 if remaining_step <= 0 else remaining_step self.assertEqual(initial_available_deposit - remaining_step, after_call_available_deposit) self.assertEqual(user_balance, after_call_user_balance) self.assertFalse(tx_results[0].step_used_details.get(self._admin)) self.assertFalse(tx_results[0].step_used_details.get( self.score_address))
class IconScoreInnerTask(object): def __init__(self, conf: 'IconConfig'): self._conf = conf self._thread_flag = ENABLE_THREAD_FLAG self._icon_service_engine = IconServiceEngine() self._open() self._thread_pool = { THREAD_INVOKE: ThreadPoolExecutor(1), THREAD_QUERY: ThreadPoolExecutor(1), THREAD_VALIDATE: ThreadPoolExecutor(1) } def _open(self): Logger.info("icon_score_service open", ICON_INNER_LOG_TAG) self._icon_service_engine.open(self._conf) def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool: return (self._thread_flag & flag) == flag def _check_icon_service_ready(self): if not self._icon_service_engine.is_reward_calculator_ready(): raise ServiceNotReadyException("Reward Calculator is not ready") @staticmethod def _log_exception(e: BaseException, tag: str = ICON_INNER_LOG_TAG) -> None: Logger.exception(str(e), tag) Logger.error(str(e), tag) @message_queue_task async def hello(self): Logger.info('hello() start', ICON_INNER_LOG_TAG) ready_future = self._icon_service_engine.get_ready_future() await ready_future if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello) else: ret = self._hello() Logger.info('hello() end', ICON_INNER_LOG_TAG) return ret def _hello(self): return self._icon_service_engine.hello() def _close(self): Logger.info(tag=_TAG, msg="_close() start") if self._icon_service_engine: self._icon_service_engine.close() self._icon_service_engine = None MessageQueueService.loop.stop() Logger.info(tag=_TAG, msg="_close() end") @message_queue_task async def close(self): Logger.info(tag=_TAG, msg="close() start") self._close() Logger.info(tag=_TAG, msg="close() end") @message_queue_task async def invoke(self, request: dict): Logger.info(f'invoke request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._invoke, request) else: return self._invoke(request) def _invoke(self, request: dict): """Process transactions in a block :param request: :return: """ response = None try: params = TypeConverter.convert(request, ParamType.INVOKE) converted_block_params = params['block'] block = Block.from_dict(converted_block_params) converted_tx_requests = params['transactions'] convert_tx_result_to_dict: bool = 'isBlockEditable' in params converted_is_block_editable = params.get('isBlockEditable', False) converted_prev_block_generator = params.get('prevBlockGenerator') converted_prev_block_validators = params.get('prevBlockValidators') converted_prev_votes = params.get('prevBlockVotes') tx_results, state_root_hash, added_transactions, main_prep_as_dict = self._icon_service_engine.invoke( block=block, tx_requests=converted_tx_requests, prev_block_generator=converted_prev_block_generator, prev_block_validators=converted_prev_block_validators, prev_block_votes=converted_prev_votes, is_block_editable=converted_is_block_editable) if convert_tx_result_to_dict: convert_tx_results = [ tx_result.to_dict(to_camel_case) for tx_result in tx_results ] else: # old version convert_tx_results = { bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case) for tx_result in tx_results } results = { 'txResults': convert_tx_results, 'stateRootHash': bytes.hex(state_root_hash), 'addedTransactions': added_transactions } if main_prep_as_dict: results["prep"] = main_prep_as_dict Logger.info(f'invoke origin response with {results}', ICON_INNER_LOG_TAG) response = MakeResponse.make_response(results) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except InvalidBaseTransactionException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: if self._icon_service_engine: self._icon_service_engine.clear_context_stack() return response @message_queue_task async def query(self, request: dict): Logger.debug(f'query request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._query, request) else: return self._query(request) def _query(self, request: dict): response = None try: method = request['method'] if method == 'debug_estimateStep': converted_request = TypeConverter.convert( request, ParamType.INVOKE_TRANSACTION) value = self._icon_service_engine.estimate_step( converted_request) else: converted_request = TypeConverter.convert( request, ParamType.QUERY) value = self._icon_service_engine.query( method, converted_request['params']) if isinstance(value, Address): value = str(value) response = MakeResponse.make_response(value) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.debug(f'query response with {response}', ICON_INNER_LOG_TAG) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def call(self, request: dict): Logger.info(f'call request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._call, request) else: return self._call(request) def _call(self, request: dict): response = None try: response = self._icon_service_engine.inner_call(request) if isinstance(response, Address): response = str(response) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'call response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def write_precommit_state(self, request: dict): Logger.info(f'write_precommit_state request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._write_precommit_state, request) else: return self._write_precommit_state(request) @staticmethod def _get_block_info_for_precommit_state( converted_block_params: dict ) -> Tuple[int, bytes, Optional[bytes]]: block_height: int = converted_block_params[ConstantKeys.BLOCK_HEIGHT] block_hash: Optional[bytes] = None if ConstantKeys.BLOCK_HASH in converted_block_params: instant_block_hash: bytes = converted_block_params[ ConstantKeys.BLOCK_HASH] else: instant_block_hash: bytes = converted_block_params[ ConstantKeys.OLD_BLOCK_HASH] block_hash = converted_block_params[ConstantKeys.NEW_BLOCK_HASH] return block_height, instant_block_hash, block_hash def _write_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block_height, instant_block_hash, block_hash = \ self._get_block_info_for_precommit_state(converted_block_params) self._icon_service_engine.commit(block_height, instant_block_hash, block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'write_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def remove_precommit_state(self, request: dict): Logger.info(f'remove_precommit_state request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._remove_precommit_state, request) else: return self._remove_precommit_state(request) def _remove_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block_height, instant_block_hash, _ = \ self._get_block_info_for_precommit_state(converted_block_params) self._icon_service_engine.remove_precommit_state( block_height, instant_block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'remove_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def rollback(self, request: dict): """Go back to the state of the given previous block :param request: :return: """ Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() start: {request}") self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() response = await loop.run_in_executor( self._thread_pool[THREAD_INVOKE], self._rollback, request) else: response = self._rollback(request) Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() end: {response}") return response def _rollback(self, request: dict) -> dict: Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() start: {request}") response = {} try: converted_params = TypeConverter.convert(request, ParamType.ROLLBACK) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH] response: dict = self._icon_service_engine.rollback( block_height, block_hash) response = MakeResponse.make_response(response) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except BaseException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() end: {response}") return response @message_queue_task async def validate_transaction(self, request: dict): Logger.debug(f'pre_validate_check request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.VALIDATE): loop = asyncio.get_event_loop() return await loop.run_in_executor( self._thread_pool[THREAD_VALIDATE], self._validate_transaction, request) else: return self._validate_transaction(request) def _validate_transaction(self, request: dict): response = None try: converted_request = TypeConverter.convert( request, ParamType.VALIDATE_TRANSACTION) self._icon_service_engine.validate_transaction(converted_request) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.debug(f'pre_validate_check response with {response}', ICON_INNER_LOG_TAG) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def change_block_hash(self, _params): self._check_icon_service_ready() return ExceptionCode.OK
class TestRecoverUsingWAL(TestIISSBase): def setUp(self): super().setUp() self.init_decentralized() self.prep_to_be_unregistered: 'EOAAccount' = self._accounts[1] self.transfer_icx(self._admin, self.prep_to_be_unregistered, 100 * 10**18) self.delegated_prep: 'EOAAccount' = self._accounts[0] self.delegate_amount: int = 100 self.delegator: 'EOAAccount' = self._accounts[PREP_MAIN_PREPS] self.transfer_icx(self._admin, self.delegator, 100 * 10**18) self.set_stake(self.delegator, self.delegate_amount) self.staker: 'EOAAccount' = self._admin self.stake_amount: int = 100 self.log_path: str = self.icon_service_engine._get_write_ahead_log_path( ) self.rc_data_path: str = os.path.join(self._state_db_root_path, IISS_DB) def tearDown(self): super().tearDown() if os.path.exists(self.log_path): os.remove(self.log_path) pass def _get_precommit_data_after_invoke(self) -> 'PrecommitData': # Invoke the transactions and changed state is going be written to WAL. # To make manipulated WAL, return precommit data # Transactions are samely used to each tests stake_tx = self.create_set_stake_tx(self.staker, self.stake_amount) unregister_tx = self.create_unregister_prep_tx( self.prep_to_be_unregistered) delegation_tx = self.create_set_delegation_tx( self.delegator, [(self.delegated_prep, self.delegate_amount)]) block, hash_list = self.make_and_req_block( [stake_tx, unregister_tx, delegation_tx]) precommit_data = self.icon_service_engine._precommit_data_manager.get( block.hash) precommit_data.block_batch.set_block_to_batch(precommit_data.revision) return precommit_data def _get_wal_writer(self, precommit_data: 'PrecommitData', is_calc_period_start_block: bool): wal_writer: 'WriteAheadLogWriter' = WriteAheadLogWriter( precommit_data.revision, max_log_count=2, block=precommit_data.block, instant_block_hash=precommit_data.block.hash) wal_writer.open(self.log_path) state_wal: 'StateWAL' = StateWAL(precommit_data.block_batch) revision: int = precommit_data.rc_db_revision if is_calc_period_start_block else -1 tx_index: int = IconScoreContext.storage.rc.get_tx_index( is_calc_period_start_block) iiss_wal: 'IissWAL' = IissWAL(precommit_data.rc_block_batch, tx_index, revision) return wal_writer, state_wal, iiss_wal @staticmethod def _write_batch_to_wal(wal_writer: 'WriteAheadLogWriter', state_wal: 'StateWAL', iiss_wal: 'IissWAL', is_calc_period_start_block: bool): if is_calc_period_start_block: wal_writer.write_state(WALState.CALC_PERIOD_START_BLOCK.value, add=False) wal_writer.write_walogable(iiss_wal) wal_writer.write_walogable(state_wal) wal_writer.flush() def _get_last_block_from_icon_service(self) -> int: return self.icon_service_engine._get_last_block().height def _get_commit_context(self, block: 'Block'): return self.icon_service_engine._context_factory.create( IconScoreContextType.DIRECT, block) def _close_and_reopen_iconservice(self): self.icon_service_engine.close() self.icon_service_engine = IconServiceEngine() self.icon_service_engine.open(self._config) def _check_the_state_and_rc_db_after_recover( self, block_height: int, is_calc_period_start_block: bool): # Check if state is updated unregister_status: int = 1 self.assertEqual(unregister_status, self.get_prep(self.prep_to_be_unregistered)["status"]) del_info: list = self.get_delegation(self.delegator)['delegations'] self.assertEqual(self.delegated_prep.address, del_info[0]['address']) self.assertEqual(self.delegate_amount, del_info[0]['value']) self.assertEqual(self.stake_amount, self.get_stake(self.staker)['stake']) # Check if rc db is updated rc_data_path: str = os.path.join(self._state_db_root_path, IISS_DB) # Get_last_rc_db: str = TestRCDatabase.get_last_rc_db_data(rc_data_path) current_rc_db = KeyValueDatabase.from_path( os.path.join(rc_data_path, "current_db")) rc_data_flag = RCDataCheckFlag(0) for rc_data in current_rc_db.iterator(): if rc_data[0][:2] == PRepsData.PREFIX: pr: 'PRepsData' = PRepsData.from_bytes(rc_data[0], rc_data[1]) if pr.block_height == block_height: rc_data_flag |= RCDataCheckFlag.PREP if rc_data[0][:2] == TxData.PREFIX: tx: 'TxData' = TxData.from_bytes(rc_data[1]) expected_index: int = -1 tx_index: int = int.from_bytes(rc_data[0][2:], 'big') if tx.type == TxType.PREP_UNREGISTER: expected_index: int = 0 rc_data_flag |= RCDataCheckFlag.UNREGISTER_TX elif tx.type == TxType.DELEGATION: expected_index: int = 1 rc_data_flag |= RCDataCheckFlag.DELEGATION_TX self.assertEqual(expected_index, tx_index) self.assertEqual(block_height, tx.block_height) if rc_data[ 0] == RewardCalcStorage.KEY_FOR_GETTING_LAST_TRANSACTION_INDEX: rc_data_flag |= RCDataCheckFlag.TX_INDEX # In case of the start of calc, should check if version, header and gv has been put correctly if is_calc_period_start_block and rc_data[ 0] == RewardCalcStorage.KEY_FOR_VERSION_AND_REVISION: rc_data_flag |= RCDataCheckFlag.VERSION if is_calc_period_start_block and rc_data[0][:2] == Header.PREFIX: rc_data_flag |= RCDataCheckFlag.HEADER if is_calc_period_start_block and rc_data[ 0][:2] == GovernanceVariable.PREFIX: rc_data_flag |= RCDataCheckFlag.GOVERNANCE if is_calc_period_start_block: self.assertEqual(RCDataCheckFlag.ALL_ON_START, rc_data_flag) self.assertTrue( os.path.isdir( os.path.join( rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}_{block_height - 1}" ))) else: self.assertEqual(RCDataCheckFlag.ALL_ON_CALC, rc_data_flag) def _check_the_db_after_recover(self, last_block_before_close: int, is_calc_period_start_block: bool): last_block_after_open: int = self._get_last_block_from_icon_service() self.assertEqual(last_block_before_close + 1, last_block_after_open) self.assertFalse(os.path.exists(self.log_path)) self._check_the_state_and_rc_db_after_recover( last_block_after_open, is_calc_period_start_block) def _remove_all_iiss_db_before_reopen(self): # For make same environment, remove all iiss_db for dir_name in os.listdir(self.rc_data_path): if dir_name.startswith(RewardCalcStorage.IISS_RC_DB_NAME_PREFIX): shutil.rmtree(os.path.join(self.rc_data_path, dir_name), ignore_errors=False, onerror=None) def test_remove_wal(self): # Success case: if WAL is incomplete, should remove the WAL last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, False) wal_writer.write_walogable(iiss_wal) wal_writer.close() self._close_and_reopen_iconservice() last_block_after_open: int = self._get_last_block_from_icon_service() self.assertFalse(os.path.exists(self.log_path)) self.assertEqual(last_block_before_close, last_block_after_open) def test_close_during_writing_rc_db_on_calc_period(self): # Success case: when iconservice is closed during writing rc data to rc db, should write state db when open self.make_blocks(self._get_last_block_from_icon_service() + 1) is_start_block: bool = False last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) context: 'IconScoreContext' = self._get_commit_context( precommit_data.block) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) wal_writer.close() # write rc data to rc db # do not write state of wal (which means overwriting the rc data to db) self.icon_service_engine._process_iiss_commit(context, precommit_data, iiss_wal, is_start_block) self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block) def test_close_after_writing_rc_db_on_calc_period(self): # Success case: when iconservice is closed after writing rc data to rc db, should write state db when open self.make_blocks(self._get_last_block_from_icon_service() + 1) is_start_block: bool = False last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) context: 'IconScoreContext' = self._get_commit_context( precommit_data.block) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) # write rc data to rc db self.icon_service_engine._process_iiss_commit(context, precommit_data, iiss_wal, is_start_block) wal_writer.write_state(WALState.WRITE_RC_DB.value, add=True) wal_writer.close() self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block) def test_close_before_change_current_to_standby_on_the_start(self): # Success case: Iconservice is closed before changing current db to standby db, # should change current db to iiss db and create new current db (That is before commit data to rc db) self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) wal_writer.close() # remove all iiss_db self._remove_all_iiss_db_before_reopen() self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block) def test_close_only_standby_exists_on_the_start(self): # Success case: Iconservice is closed after changing current db to standby db, # should change stanby db to iiss db and create new current db (That is before commit data to rc db) self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) wal_writer.close() # Change the current_db to standby_db RewardCalcStorage.rename_current_db_to_standby_db( self.rc_data_path, last_block_before_close) # Remove all iiss_db self._remove_all_iiss_db_before_reopen() self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block) def test_close_standby_and_current_exists_on_the_start(self): # Success case: Iconservice is closed after changing current db to standby db and create new current db, # should change standby db to iiss db (That is before commit data to rc db) self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) wal_writer.close() # Change the current_db to standby_db RewardCalcStorage.rename_current_db_to_standby_db( self.rc_data_path, last_block_before_close) RewardCalcStorage.create_current_db(self.rc_data_path) # Remove all iiss_db self._remove_all_iiss_db_before_reopen() self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block) def test_close_before_sending_calculate_on_the_start(self): # Success case: Iconservice is closed after changing current db to iiss db and create new current db, # should not recover. self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke( ) context: 'IconScoreContext' = self._get_commit_context( precommit_data.block) wal_writer, state_wal, iiss_wal = self._get_wal_writer( precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) # Finish the iiss commit standby_db_path = self.icon_service_engine._process_iiss_commit( context, precommit_data, iiss_wal, is_start_block) wal_writer.write_state(WALState.WRITE_RC_DB.value, add=True) wal_writer.flush() # Finish the state commit self.icon_service_engine._process_state_commit(context, precommit_data, state_wal) wal_writer.write_state(WALState.WRITE_STATE_DB.value, add=True) wal_writer.flush() wal_writer.close() # Remove all iiss_db self._remove_all_iiss_db_before_reopen() # Change the standby db to iiss_db RewardCalcStorage.rename_standby_db_to_iiss_db(standby_db_path.path) self._close_and_reopen_iconservice() self._check_the_db_after_recover(last_block_before_close, is_start_block)
class IconServiceSyncer(object): _TAG = "SYNC" def __init__(self): self._block_reader = BlockDatabaseReader() self._engine = IconServiceEngine() def open( self, config_path: str, fee: bool = True, audit: bool = True, deployer_whitelist: bool = False, score_package_validator: bool = False, builtin_score_owner: str = "", ): conf = IconConfig("", default_icon_config) if config_path != "": conf.load(config_path) conf.update_conf({ "builtinScoreOwner": builtin_score_owner, "service": { "fee": fee, "audit": audit, "scorePackageValidator": score_package_validator, "deployerWhiteList": deployer_whitelist, }, }) Logger.load_config(conf) self._engine.open(conf) def run(self, *args, **kwargs) -> int: Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}") loop = asyncio.get_event_loop() future = asyncio.Future() try: asyncio.ensure_future( self._wait_for_complete(future, *args, **kwargs)) loop.run_until_complete(future) finally: for task in asyncio.Task.all_tasks(): task.cancel() loop.close() ret = future.result() Logger.debug(tag=self._TAG, msg=f"run() end: {ret}") return ret async def _wait_for_complete(self, result_future: asyncio.Future, *args, **kwargs): Logger.debug(tag=self._TAG, msg="_wait_for_complete() start") # Wait for rc to be ready future = self._engine.get_ready_future() await future with ThreadPoolExecutor(max_workers=1) as executor: # Call IconServiceEngine.hello() f = executor.submit(self._hello) future = asyncio.wrap_future(f) await future # Start to sync blocks f = executor.submit(self._run, *args, **kwargs) f = asyncio.wrap_future(f) for fut in asyncio.as_completed([f]): try: ret = await fut except Exception as exc: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) result_future.set_exception(exc) else: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end1") result_future.set_result(ret) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end2") def _hello(self): self._engine.hello() def _run( self, db_path: str, channel: str, start_height: int = 0, count: int = 99999999, stop_on_error: bool = True, no_commit: bool = False, backup_period: int = 0, write_precommit_data: bool = False, print_block_height: int = 1, iiss_db_backup_path: Optional[str] = None, ) -> int: """Begin to synchronize IconServiceEngine with blocks from loopchain db :param db_path: loopchain db path :param channel: channel name used as a key to get commit_state in loopchain db :param start_height: start height to sync :param count: The number of blocks to sync :param stop_on_error: If error happens, stop syncing :param no_commit: Do not commit :param backup_period: state backup period in block :param write_precommit_data: :param print_block_height: print every this block height :return: 0(success), otherwise(error) """ Logger.debug(tag=self._TAG, msg="_run() start") word_detector = WordDetector( filename="iconservice.log", block_word=r"CALCULATE\(", release_word=r"CALCULATE_DONE\(", ) ret: int = 0 self._block_reader.open(db_path) print("block_height | commit_state | state_root_hash | tx_count") prev_block: Optional["Block"] = None prev_loopchain_block: Optional["LoopchainBlock"] = None main_preps: Optional['NodeContainer'] = None next_main_preps: Optional["NodeContainer"] = None if start_height > 0: prev_block_dict = self._block_reader.get_block_by_block_height( start_height - 1) prev_loopchain_block = LoopchainBlock.from_dict(prev_block_dict) # init main_preps preps: list = self._block_reader.load_main_preps(prev_block_dict) main_preps: Optional['NodeContainer'] = NodeContainer.from_list( preps=preps) # when sync from the first block of term, have to initialize next_main_preps here # in that case, invoke_result[3] will be None on first block and can not update next_main_preps block_dict = self._block_reader.get_block_by_block_height( start_height) loopchain_block = LoopchainBlock.from_dict(block_dict) block = _create_iconservice_block(loopchain_block) if self._check_calculation_block(block): preps: list = self._block_reader.load_main_preps(block_dict) next_main_preps: Optional[ 'NodeContainer'] = NodeContainer.from_list(preps=preps) end_height = start_height + count - 1 for height in range(start_height, start_height + count): block_dict: dict = self._block_reader.get_block_by_block_height( height) if block_dict is None: print(f"last block: {height - 1}") break loopchain_block: 'LoopchainBlock' = LoopchainBlock.from_dict( block_dict) block: 'Block' = _create_iconservice_block(loopchain_block) tx_requests: list = create_transaction_requests(loopchain_block) prev_block_generator: Optional[ "Address"] = prev_loopchain_block.leader if prev_loopchain_block else None prev_block_validators: Optional[ List["Address"]] = _create_block_validators( block_dict, prev_block_generator) prev_block_votes: Optional[List[Tuple[ "Address", int]]] = _create_prev_block_votes(block_dict, prev_block_generator, main_preps) if prev_block is not None and prev_block.hash != block.prev_hash: raise Exception() invoke_result = self._engine.invoke( block, tx_requests, prev_block_generator, prev_block_validators, prev_block_votes, ) tx_results, state_root_hash = invoke_result[0], invoke_result[1] main_preps_as_dict: Optional[Dict] = invoke_result[3] commit_state: bytes = self._block_reader.get_commit_state( block_dict, channel) # "commit_state" is the field name of state_root_hash in loopchain block if (height - start_height) % print_block_height == 0: print( f"{height} | {commit_state.hex()[:6]} | {state_root_hash.hex()[:6]} | {len(tx_requests)}" ) if write_precommit_data: self._print_precommit_data(block) try: if stop_on_error: if commit_state: if commit_state != state_root_hash: raise Exception() if height > 0 and not self._check_invoke_result( tx_results): raise Exception() except Exception as e: logging.exception(e) self._print_precommit_data(block) ret: int = 1 break is_calculation_block = self._check_calculation_block(block) if is_calculation_block: word_detector.start() time.sleep(0.5) if iiss_db_backup_path is not None: self._backup_iiss_db(iiss_db_backup_path, block.height) # If no_commit is set to True, the config only affects to the last block to commit if height < end_height: while word_detector.get_hold(): time.sleep(0.5) # Call IconServiceEngine.commit() with a block if not no_commit: self._commit(block) while word_detector.get_hold(): time.sleep(0.5) # Prepare the next iteration self._backup_state_db(block, backup_period) prev_block = block prev_loopchain_block = loopchain_block if next_main_preps: main_preps = next_main_preps next_main_preps = None if main_preps_as_dict is not None: next_main_preps = NodeContainer.from_dict(main_preps_as_dict) self._block_reader.close() word_detector.stop() Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}") return ret def _commit(self, block: "Block"): if "block" in inspect.signature(self._engine.commit).parameters: self._engine.commit(block) else: self._engine.commit(block.height, block.hash, block.hash) def _backup_iiss_db(self, iiss_db_backup_path: Optional[str], block_height: int): iiss_db_path: str = os.path.join(self._engine._state_db_root_path, "iiss") with os.scandir(iiss_db_path) as it: for entry in it: if entry.is_dir( ) and entry.name == Storage.CURRENT_IISS_DB_NAME: dst_path: str = os.path.join( iiss_db_backup_path, f"{Storage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}", ) if os.path.exists(dst_path): shutil.rmtree(dst_path) shutil.copytree(entry.path, dst_path) break def _check_invoke_result(self, tx_results: list): """Compare the transaction results from IconServiceEngine with the results stored in loopchain db If transaction result is not compatible to protocol v3, pass it :param tx_results: the transaction results that IconServiceEngine.invoke() returns :return: True(same) False(different) """ for tx_result in tx_results: tx_info_in_db: dict = self._block_reader.get_transaction_result_by_hash( tx_result.tx_hash.hex()) tx_result_in_db = tx_info_in_db["result"] # tx_v2 dose not have transaction result_v3 if "status" not in tx_result_in_db: continue # information extracted from db status: int = int(tx_result_in_db["status"], 16) tx_hash: bytes = bytes.fromhex(tx_result_in_db["txHash"]) step_used: int = int(tx_result_in_db["stepUsed"], 16) step_price: int = int(tx_result_in_db["stepPrice"], 16) event_logs: list = tx_result_in_db["eventLogs"] step: int = step_used * step_price if tx_hash != tx_result.tx_hash: print(f"tx_hash: {tx_hash.hex()} != {tx_result.tx_hash.hex()}") return False if status != tx_result.status: print(f"status: {status} != {tx_result.status}") return False if step_used != tx_result.step_used: print(f"step_used: {step_used} != {tx_result.step_used}") return False tx_result_step: int = tx_result.step_used * tx_result.step_price if step != tx_result_step: print(f"step: {step} != {tx_result_step}") return False if step_price != tx_result.step_price: print(f"step_price: {step_price} != {tx_result.step_price}") return False if not self._check_event_logs(event_logs, tx_result.event_logs): return False return True @staticmethod def _check_event_logs(event_logs_in_db: list, event_logs_in_tx_result: list): if event_logs_in_db is None: event_logs_in_db = [] if event_logs_in_tx_result is None: event_logs_in_tx_result = [] if len(event_logs_in_db) != len(event_logs_in_tx_result): return False for event_log, _tx_result_event_log in zip(event_logs_in_db, event_logs_in_tx_result): tx_result_event_log: dict = _tx_result_event_log.to_dict() # convert Address to str if "score_address" in tx_result_event_log: score_address: "Address" = tx_result_event_log["score_address"] del tx_result_event_log["score_address"] tx_result_event_log["scoreAddress"] = str(score_address) # convert Address objects to str objects in 'indexes' indexed: list = tx_result_event_log["indexed"] for i in range(len(indexed)): value = indexed[i] indexed[i] = object_to_str(value) data: list = tx_result_event_log["data"] for i in range(len(data)): value = data[i] data[i] = object_to_str(value) if event_log != tx_result_event_log: print(f"{event_log} != {tx_result_event_log}") return False return True def _print_precommit_data(self, block: "Block"): """Print the latest updated states stored in IconServiceEngine :return: """ precommit_data_manager: PrecommitDataManager = getattr( self._engine, "_precommit_data_manager") precommit_data: PrecommitData = precommit_data_manager.get(block.hash) block_batch: BlockBatch = precommit_data.block_batch state_root_hash: bytes = block_batch.digest() filename = f"{block.height}-precommit-data.txt" with open(filename, "wt") as f: for i, key in enumerate(block_batch): value: "TransactionBatchValue" = block_batch[key] if value: hex_value = value.value.hex( ) if value.value is not None else None include_state_root_hash = value.include_state_root_hash line = f"{i}: {key.hex()} - {hex_value}, {include_state_root_hash}" else: line = f"{i}: {key.hex()} - None" print(line) f.write(f"{line}\n") f.write(f"state_root_hash: {state_root_hash.hex()}\n") def _check_calculation_block(self, block: "Block") -> bool: """check calculation block""" context = IconScoreContext(IconScoreContextType.DIRECT) revision = self._engine._get_revision_from_rc(context) context.block = block if revision < Revision.IISS.value: return False start_block = context.engine.iiss.get_start_block_of_calc(context) return start_block == block.height @staticmethod def _backup_state_db(block: "Block", backup_period: int): if backup_period <= 0: return if block.height == 0: return if block.height % backup_period == 0: print(f"----------- Backup statedb: {block.height} ------------") dirname: str = f"block-{'%09d' % block.height}" for basename in (".score", ".statedb"): try: shutil.copytree(basename, f"{dirname}/{basename}/") except FileExistsError: pass def close(self): pass