class IconServiceSyncer(object): _TAG = "SYNC" def __init__(self): self._block_reader = BinBlockDatabaseReader() self._engine = IconServiceEngine() self._timer = Timer() def open( self, config_path: str, fee: bool = True, audit: bool = True, deployer_whitelist: bool = False, score_package_validator: bool = False, builtin_score_owner: str = "", ): conf = IconConfig("", default_icon_config) if config_path != "": conf.load(config_path) conf.update_conf({ "builtinScoreOwner": builtin_score_owner, "service": { "fee": fee, "audit": audit, "scorePackageValidator": score_package_validator, "deployerWhiteList": deployer_whitelist, }, }) Logger.load_config(conf) self._engine.open(conf) def run(self, *args, **kwargs) -> int: Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}") loop = asyncio.get_event_loop() future = asyncio.Future() try: asyncio.ensure_future( self._wait_for_complete(future, *args, **kwargs)) loop.run_until_complete(future) finally: for task in asyncio.Task.all_tasks(): task.cancel() loop.close() ret = future.result() Logger.debug(tag=self._TAG, msg=f"run() end: {ret}") return ret async def _wait_for_complete(self, result_future: asyncio.Future, *args, **kwargs): Logger.debug(tag=self._TAG, msg="_wait_for_complete() start") # Wait for rc to be ready future = self._engine.get_ready_future() await future with ThreadPoolExecutor(max_workers=1) as executor: # Call IconServiceEngine.hello() f = executor.submit(self._hello) future = asyncio.wrap_future(f) await future # Start to sync blocks f = executor.submit(self._run, *args, **kwargs) f = asyncio.wrap_future(f) for fut in asyncio.as_completed([f]): try: ret = await fut except Exception as exc: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) result_future.set_exception(exc) else: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end1") result_future.set_result(ret) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end2") def _hello(self): self._engine.hello() def _run( self, db_path: str, channel: str, start_height: int = 0, count: int = 99_999_999, stop_on_error: bool = True, no_commit: bool = False, backup_period: int = 0, write_precommit_data: bool = False, print_block_height: int = 1, iiss_db_backup_path: Optional[str] = None, ) -> int: """Begin to synchronize IconServiceEngine with blocks from loopchain db :param db_path: loopchain db path :param channel: channel name used as a key to get commit_state in loopchain db :param start_height: start height to sync :param count: The number of blocks to sync :param stop_on_error: If error happens, stop syncing :param no_commit: Do not commit :param backup_period: state backup period in block :param write_precommit_data: :param print_block_height: print every this block height :return: 0(success), otherwise(error) """ Logger.debug(tag=self._TAG, msg="_run() start") word_detector = WordDetector( filename="iconservice.log", block_word=r"CALCULATE\(", release_word=r"CALCULATE_DONE\(", ) ret: int = 0 self._block_reader.open(db_path) print("block_height | commit_state | state_root_hash | tx_count") prev_bin_block: Optional["BinBlock"] = None prev_block: Optional["Block"] = None main_preps: Optional['NodeContainer'] = None next_main_preps: Optional["NodeContainer"] = None if start_height > 0: prev_bin_block: Optional[ BinBlock] = self._block_reader.get_block_by_height( start_height - 1) # init main_preps preps: list = self._block_reader.load_main_preps( reps_hash=prev_bin_block.reps_hash) main_preps: Optional['NodeContainer'] = NodeContainer.from_list( preps=preps) # when sync from the first block of term, have to initialize next_main_preps here # in that case, invoke_result[3] will be None on first block and can not update next_main_preps cur_bin_block: Optional[ BinBlock] = self._block_reader.get_block_by_height( start_height) block: "Block" = create_iconservice_block(cur_bin_block) if self._check_calculation_block(block): preps: list = self._block_reader.load_main_preps( reps_hash=cur_bin_block.reps_hash) next_main_preps: Optional[ 'NodeContainer'] = NodeContainer.from_list(preps=preps) end_height = start_height + count - 1 self._timer.start() for height in range(start_height, start_height + count): bin_block = self._block_reader.get_block_by_height(height) if bin_block is None: print(f"last block: {height - 1}") break block: "Block" = create_iconservice_block(bin_block) tx_requests: List[Dict[str, Any]] = create_transaction_requests( bin_block.transactions) prev_block_generator: Optional[ "Address"] = prev_bin_block.leader if prev_bin_block else None prev_block_validators: Optional[ List["Address"]] = create_block_validators( bin_block.prev_votes, prev_block_generator) prev_block_votes: Optional[List[ Tuple["Address", int]]] = create_prev_block_votes(bin_block.prev_votes, prev_block_generator, main_preps) Logger.info(tag=self._TAG, msg=f"prev_block_generator={prev_block_generator}") Logger.info(tag=self._TAG, msg=f"prev_block_validators={prev_block_validators}") Logger.info(tag=self._TAG, msg=f"prev_block_votes={prev_block_votes}") if prev_block is not None and prev_block.hash != block.prev_hash: raise Exception(f"Invalid prev_block_hash: height={height}") invoke_result = self._engine.invoke( block, tx_requests, prev_block_generator, prev_block_validators, prev_block_votes, ) tx_results, state_root_hash = invoke_result[0], invoke_result[1] main_preps_as_dict: Optional[Dict] = invoke_result[3] commit_state: bytes = bin_block.state_hash # "commit_state" is the field name of state_root_hash in loopchain block if (height - start_height) % print_block_height == 0: self._print_status(height, start_height, count, commit_state, state_root_hash, len(tx_requests)) if write_precommit_data: self._print_precommit_data(block) try: if stop_on_error: if commit_state: if commit_state != state_root_hash: raise Exception("state_root_hash mismatch") if height > 0 and not self._check_invoke_result( tx_results): raise Exception("tx_result mismatch") except Exception as e: logging.exception(e) self._print_precommit_data(block) ret: int = 1 break is_calculation_block = self._check_calculation_block(block) if is_calculation_block: word_detector.start() time.sleep(0.5) if iiss_db_backup_path is not None: self._backup_iiss_db(iiss_db_backup_path, block.height) # If no_commit is set to True, the config only affects to the last block to commit if not no_commit or height < end_height: while word_detector.get_hold(): time.sleep(0.5) # Call IconServiceEngine.commit() with a block self._commit(block) while word_detector.get_hold(): time.sleep(0.5) # Prepare the next iteration self._backup_state_db(block, backup_period) prev_block: 'Block' = block prev_bin_block: 'BinBlock' = bin_block if next_main_preps: main_preps = next_main_preps next_main_preps = None if main_preps_as_dict is not None: next_main_preps = NodeContainer.from_dict(main_preps_as_dict) self._block_reader.close() word_detector.stop() Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}") return ret
class IconScoreInnerTask(object): def __init__(self, conf: dict): self._conf = conf self._thread_flag = ENABLE_THREAD_FLAG self._icon_service_engine = IconServiceEngine() self._open() self._thread_pool = { THREAD_INVOKE: ThreadPoolExecutor(1), THREAD_STATUS: ThreadPoolExecutor(1), THREAD_QUERY: ThreadPoolExecutor(1), THREAD_ESTIMATE: ThreadPoolExecutor(1), THREAD_VALIDATE: ThreadPoolExecutor(1) } def _open(self): Logger.info(tag=_TAG, msg="_open() start") self._icon_service_engine.open(self._conf) Logger.info(tag=_TAG, msg="_open() end") def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool: return (self._thread_flag & flag) == flag def _check_icon_service_ready(self): if not self._icon_service_engine.is_reward_calculator_ready(): raise ServiceNotReadyException("Reward Calculator is not ready") @staticmethod def _log_exception(e: BaseException, tag: str) -> None: Logger.exception(str(e), tag) Logger.error(str(e), tag) @message_queue_task async def hello(self): Logger.info(tag=_TAG, msg='hello() start') ready_future = self._icon_service_engine.get_ready_future() await ready_future if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello) else: ret = self._hello() Logger.info(tag=_TAG, msg='hello() end') return ret def _hello(self): return self._icon_service_engine.hello() def cleanup(self): Logger.info(tag=_TAG, msg="cleanup() start") # shutdown thread pool executors for executor in self._thread_pool.values(): executor.shutdown() # close ICON Service if self._icon_service_engine: self._icon_service_engine.close() self._icon_service_engine = None Logger.info(tag=_TAG, msg="cleanup() end") @message_queue_task async def close(self): Logger.info(tag=_TAG, msg="close() stop event loop") self._close() @staticmethod def _close(): asyncio.get_event_loop().stop() @message_queue_task async def invoke(self, request: dict) -> dict: Logger.debug(tag=_TAG, msg=f'invoke() start') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret: dict = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._invoke, request) else: ret: dict = self._invoke(request) Logger.debug(tag=_TAG, msg=f'invoke() end') return ret def _invoke(self, request: dict): """Process transactions in a block :param request: :return: """ Logger.info(tag=_TAG, msg=f'INVOKE Request: {request}') try: params = TypeConverter.convert(request, ParamType.INVOKE) converted_block_params = params['block'] block = Block.from_dict(converted_block_params) Logger.info(tag=_TAG, msg=f'INVOKE: BH={block.height}') converted_tx_requests = params['transactions'] convert_tx_result_to_dict: bool = 'isBlockEditable' in params converted_is_block_editable = params.get('isBlockEditable', False) converted_prev_block_generator = params.get('prevBlockGenerator') converted_prev_block_validators = params.get('prevBlockValidators') converted_prev_votes = params.get('prevBlockVotes') tx_results, state_root_hash, added_transactions, next_preps, is_shutdown = \ self._icon_service_engine.invoke( block=block, tx_requests=converted_tx_requests, prev_block_generator=converted_prev_block_generator, prev_block_validators=converted_prev_block_validators, prev_block_votes=converted_prev_votes, is_block_editable=converted_is_block_editable ) if convert_tx_result_to_dict: convert_tx_results = [tx_result.to_dict(to_camel_case) for tx_result in tx_results] else: # old version convert_tx_results = {bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case) for tx_result in tx_results} results = { 'txResults': convert_tx_results, 'stateRootHash': bytes.hex(state_root_hash), 'addedTransactions': added_transactions } if next_preps: results["prep"] = next_preps if is_shutdown: results["is_shutdown"] = True response = MakeResponse.make_response(results) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except InvalidBaseTransactionException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) finally: if self._icon_service_engine: self._icon_service_engine.clear_context_stack() Logger.info(tag=_TAG, msg=f'INVOKE Response: {json.dumps(response, cls=BytesToHexJSONEncoder)}') return response @message_queue_task async def query(self, request: dict) -> dict: try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) return await self._get_query_response(request) async def _get_query_response(self, request: dict) -> dict: try: value = await self._execute_query(request) if isinstance(value, Address): value = str(value) response = MakeResponse.make_response(value) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._icon_service_engine.clear_context_stack() return response async def _execute_query(self, request: dict): method_name: str = request['method'] if method_name == RPCMethod.DEBUG_ESTIMATE_STEP: method: callable = self._estimate args = [request] else: method: callable = self._query args = [request, method_name] if self._is_thread_flag_on(EnableThreadFlag.QUERY): return await asyncio.get_event_loop(). \ run_in_executor(self._thread_pool[QUERY_THREAD_MAPPER[method_name]], method, *args) else: return method(*args) def _estimate(self, request: dict): converted_request = TypeConverter.convert(request, ParamType.INVOKE_TRANSACTION) return self._icon_service_engine.estimate_step(converted_request) def _query(self, request: dict, method: str): converted_request = TypeConverter.convert(request, ParamType.QUERY) return self._icon_service_engine.query(method, converted_request['params']) @message_queue_task async def call(self, request: dict): Logger.info(tag=_TAG, msg=f'call() start: {request}') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._call, request) else: ret = self._call(request) Logger.info(tag=_TAG, msg=f'call() end: {ret}') return ret def _call(self, request: dict): try: response = self._icon_service_engine.inner_call(request) if isinstance(response, Address): response = str(response) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) return response @message_queue_task async def write_precommit_state(self, request: dict): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._write_precommit_state, request) else: ret = self._write_precommit_state(request) return ret def _write_precommit_state(self, request: dict) -> dict: Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Request: {request}') try: converted_params = TypeConverter.convert(request, ParamType.WRITE_PRECOMMIT) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] instant_block_hash: bytes = converted_params[ConstantKeys.OLD_BLOCK_HASH] block_hash = converted_params[ConstantKeys.NEW_BLOCK_HASH] Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE: ' f'BH={block_height} ' f'instant_block_hash={bytes_to_hex(instant_block_hash)} ' f'block_hash={bytes_to_hex(block_hash)}') self._icon_service_engine.commit(block_height, instant_block_hash, block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Response: {response}') return response @message_queue_task async def remove_precommit_state(self, request: dict): Logger.info(tag=_TAG, msg=f'remove_precommit_state() start') try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) """ Unused API """ return {} @message_queue_task async def rollback(self, request: dict): """Go back to the state of the given previous block :param request: :return: """ Logger.info(tag=_TAG, msg=f"rollback() start") try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() response = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._rollback, request) else: response = self._rollback(request) Logger.info(tag=_TAG, msg=f"rollback() end") return response def _rollback(self, request: dict) -> dict: Logger.info(tag=_TAG, msg=f"ROLLBACK Request: {request}") try: converted_params = TypeConverter.convert(request, ParamType.ROLLBACK) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH] Logger.info(tag=_TAG, msg=f"ROLLBACK: BH={block_height} block_hash={bytes_to_hex(block_hash)}") response: dict = self._icon_service_engine.rollback(block_height, block_hash) response = MakeResponse.make_response(response) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except BaseException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) Logger.info(tag=_TAG, msg=f"ROLLBACK Response: {response}") return response @message_queue_task async def validate_transaction(self, request: dict): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) if self._is_thread_flag_on(EnableThreadFlag.VALIDATE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_VALIDATE], self._validate_transaction, request) else: return self._validate_transaction(request) def _validate_transaction(self, request: dict): try: Logger.info(tag=_TAG, msg=f'validate_transaction Request: {request}') converted_request = TypeConverter.convert(request, ParamType.VALIDATE_TRANSACTION) self._icon_service_engine.validate_transaction(converted_request, request) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, _TAG) response = MakeResponse.make_error_response(icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def change_block_hash(self, _params): try: self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) return ExceptionCode.OK @message_queue_task async def dos_guard(self, params: dict) -> dict: try: Logger.info(tag=_TAG, msg=f'dos_guard: params: {params}') self._check_icon_service_ready() except ServiceNotReadyException as e: return MakeResponse.make_error_response(e.code, str(e)) try: self._icon_service_engine.dos_guard.run(_from=params["from"]) response = MakeResponse.make_response(ExceptionCode.OK) except Exception as e: self._log_exception(e, _TAG) response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) return response
class IconScoreInnerTask(object): def __init__(self, conf: 'IconConfig'): self._conf = conf self._thread_flag = ENABLE_THREAD_FLAG self._icon_service_engine = IconServiceEngine() self._open() self._thread_pool = { THREAD_INVOKE: ThreadPoolExecutor(1), THREAD_QUERY: ThreadPoolExecutor(1), THREAD_VALIDATE: ThreadPoolExecutor(1) } def _open(self): Logger.info("icon_score_service open", ICON_INNER_LOG_TAG) self._icon_service_engine.open(self._conf) def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool: return (self._thread_flag & flag) == flag def _check_icon_service_ready(self): if not self._icon_service_engine.is_reward_calculator_ready(): raise ServiceNotReadyException("Reward Calculator is not ready") @staticmethod def _log_exception(e: BaseException, tag: str = ICON_INNER_LOG_TAG) -> None: Logger.exception(str(e), tag) Logger.error(str(e), tag) @message_queue_task async def hello(self): Logger.info('hello() start', ICON_INNER_LOG_TAG) ready_future = self._icon_service_engine.get_ready_future() await ready_future if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello) else: ret = self._hello() Logger.info('hello() end', ICON_INNER_LOG_TAG) return ret def _hello(self): return self._icon_service_engine.hello() def _close(self): Logger.info(tag=_TAG, msg="_close() start") if self._icon_service_engine: self._icon_service_engine.close() self._icon_service_engine = None MessageQueueService.loop.stop() Logger.info(tag=_TAG, msg="_close() end") @message_queue_task async def close(self): Logger.info(tag=_TAG, msg="close() start") self._close() Logger.info(tag=_TAG, msg="close() end") @message_queue_task async def invoke(self, request: dict): Logger.info(f'invoke request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._invoke, request) else: return self._invoke(request) def _invoke(self, request: dict): """Process transactions in a block :param request: :return: """ response = None try: params = TypeConverter.convert(request, ParamType.INVOKE) converted_block_params = params['block'] block = Block.from_dict(converted_block_params) converted_tx_requests = params['transactions'] convert_tx_result_to_dict: bool = 'isBlockEditable' in params converted_is_block_editable = params.get('isBlockEditable', False) converted_prev_block_generator = params.get('prevBlockGenerator') converted_prev_block_validators = params.get('prevBlockValidators') converted_prev_votes = params.get('prevBlockVotes') tx_results, state_root_hash, added_transactions, main_prep_as_dict = self._icon_service_engine.invoke( block=block, tx_requests=converted_tx_requests, prev_block_generator=converted_prev_block_generator, prev_block_validators=converted_prev_block_validators, prev_block_votes=converted_prev_votes, is_block_editable=converted_is_block_editable) if convert_tx_result_to_dict: convert_tx_results = [ tx_result.to_dict(to_camel_case) for tx_result in tx_results ] else: # old version convert_tx_results = { bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case) for tx_result in tx_results } results = { 'txResults': convert_tx_results, 'stateRootHash': bytes.hex(state_root_hash), 'addedTransactions': added_transactions } if main_prep_as_dict: results["prep"] = main_prep_as_dict Logger.info(f'invoke origin response with {results}', ICON_INNER_LOG_TAG) response = MakeResponse.make_response(results) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except InvalidBaseTransactionException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: if self._icon_service_engine: self._icon_service_engine.clear_context_stack() return response @message_queue_task async def query(self, request: dict): Logger.debug(f'query request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._query, request) else: return self._query(request) def _query(self, request: dict): response = None try: method = request['method'] if method == 'debug_estimateStep': converted_request = TypeConverter.convert( request, ParamType.INVOKE_TRANSACTION) value = self._icon_service_engine.estimate_step( converted_request) else: converted_request = TypeConverter.convert( request, ParamType.QUERY) value = self._icon_service_engine.query( method, converted_request['params']) if isinstance(value, Address): value = str(value) response = MakeResponse.make_response(value) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.debug(f'query response with {response}', ICON_INNER_LOG_TAG) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def call(self, request: dict): Logger.info(f'call request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.QUERY): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_QUERY], self._call, request) else: return self._call(request) def _call(self, request: dict): response = None try: response = self._icon_service_engine.inner_call(request) if isinstance(response, Address): response = str(response) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'call response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def write_precommit_state(self, request: dict): Logger.info(f'write_precommit_state request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._write_precommit_state, request) else: return self._write_precommit_state(request) @staticmethod def _get_block_info_for_precommit_state( converted_block_params: dict ) -> Tuple[int, bytes, Optional[bytes]]: block_height: int = converted_block_params[ConstantKeys.BLOCK_HEIGHT] block_hash: Optional[bytes] = None if ConstantKeys.BLOCK_HASH in converted_block_params: instant_block_hash: bytes = converted_block_params[ ConstantKeys.BLOCK_HASH] else: instant_block_hash: bytes = converted_block_params[ ConstantKeys.OLD_BLOCK_HASH] block_hash = converted_block_params[ConstantKeys.NEW_BLOCK_HASH] return block_height, instant_block_hash, block_hash def _write_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block_height, instant_block_hash, block_hash = \ self._get_block_info_for_precommit_state(converted_block_params) self._icon_service_engine.commit(block_height, instant_block_hash, block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'write_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def remove_precommit_state(self, request: dict): Logger.info(f'remove_precommit_state request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._remove_precommit_state, request) else: return self._remove_precommit_state(request) def _remove_precommit_state(self, request: dict): response = None try: converted_block_params = TypeConverter.convert( request, ParamType.WRITE_PRECOMMIT) block_height, instant_block_hash, _ = \ self._get_block_info_for_precommit_state(converted_block_params) self._icon_service_engine.remove_precommit_state( block_height, instant_block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(f'remove_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response @message_queue_task async def rollback(self, request: dict): """Go back to the state of the given previous block :param request: :return: """ Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() start: {request}") self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): loop = asyncio.get_event_loop() response = await loop.run_in_executor( self._thread_pool[THREAD_INVOKE], self._rollback, request) else: response = self._rollback(request) Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() end: {response}") return response def _rollback(self, request: dict) -> dict: Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() start: {request}") response = {} try: converted_params = TypeConverter.convert(request, ParamType.ROLLBACK) block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH] response: dict = self._icon_service_engine.rollback( block_height, block_hash) response = MakeResponse.make_response(response) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) self._close() except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except BaseException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() end: {response}") return response @message_queue_task async def validate_transaction(self, request: dict): Logger.debug(f'pre_validate_check request with {request}', ICON_INNER_LOG_TAG) self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.VALIDATE): loop = asyncio.get_event_loop() return await loop.run_in_executor( self._thread_pool[THREAD_VALIDATE], self._validate_transaction, request) else: return self._validate_transaction(request) def _validate_transaction(self, request: dict): response = None try: converted_request = TypeConverter.convert( request, ParamType.VALIDATE_TRANSACTION) self._icon_service_engine.validate_transaction(converted_request) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) except IconServiceBaseException as icon_e: self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( icon_e.code, icon_e.message) except Exception as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) response = MakeResponse.make_error_response( ExceptionCode.SYSTEM_ERROR, str(e)) finally: Logger.debug(f'pre_validate_check response with {response}', ICON_INNER_LOG_TAG) self._icon_service_engine.clear_context_stack() return response @message_queue_task async def change_block_hash(self, _params): self._check_icon_service_ready() return ExceptionCode.OK
class IconServiceSyncer(object): _TAG = "SYNC" def __init__(self): self._block_reader = BlockDatabaseReader() self._engine = IconServiceEngine() def open( self, config_path: str, fee: bool = True, audit: bool = True, deployer_whitelist: bool = False, score_package_validator: bool = False, builtin_score_owner: str = "", ): conf = IconConfig("", default_icon_config) if config_path != "": conf.load(config_path) conf.update_conf({ "builtinScoreOwner": builtin_score_owner, "service": { "fee": fee, "audit": audit, "scorePackageValidator": score_package_validator, "deployerWhiteList": deployer_whitelist, }, }) Logger.load_config(conf) self._engine.open(conf) def run(self, *args, **kwargs) -> int: Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}") loop = asyncio.get_event_loop() future = asyncio.Future() try: asyncio.ensure_future( self._wait_for_complete(future, *args, **kwargs)) loop.run_until_complete(future) finally: for task in asyncio.Task.all_tasks(): task.cancel() loop.close() ret = future.result() Logger.debug(tag=self._TAG, msg=f"run() end: {ret}") return ret async def _wait_for_complete(self, result_future: asyncio.Future, *args, **kwargs): Logger.debug(tag=self._TAG, msg="_wait_for_complete() start") # Wait for rc to be ready future = self._engine.get_ready_future() await future with ThreadPoolExecutor(max_workers=1) as executor: # Call IconServiceEngine.hello() f = executor.submit(self._hello) future = asyncio.wrap_future(f) await future # Start to sync blocks f = executor.submit(self._run, *args, **kwargs) f = asyncio.wrap_future(f) for fut in asyncio.as_completed([f]): try: ret = await fut except Exception as exc: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) result_future.set_exception(exc) else: self._engine.close() # Wait to stop ipc_server for 1s await asyncio.sleep(1) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end1") result_future.set_result(ret) Logger.debug(tag=self._TAG, msg="_wait_for_complete() end2") def _hello(self): self._engine.hello() def _run( self, db_path: str, channel: str, start_height: int = 0, count: int = 99999999, stop_on_error: bool = True, no_commit: bool = False, backup_period: int = 0, write_precommit_data: bool = False, print_block_height: int = 1, iiss_db_backup_path: Optional[str] = None, ) -> int: """Begin to synchronize IconServiceEngine with blocks from loopchain db :param db_path: loopchain db path :param channel: channel name used as a key to get commit_state in loopchain db :param start_height: start height to sync :param count: The number of blocks to sync :param stop_on_error: If error happens, stop syncing :param no_commit: Do not commit :param backup_period: state backup period in block :param write_precommit_data: :param print_block_height: print every this block height :return: 0(success), otherwise(error) """ Logger.debug(tag=self._TAG, msg="_run() start") word_detector = WordDetector( filename="iconservice.log", block_word=r"CALCULATE\(", release_word=r"CALCULATE_DONE\(", ) ret: int = 0 self._block_reader.open(db_path) print("block_height | commit_state | state_root_hash | tx_count") prev_block: Optional["Block"] = None prev_loopchain_block: Optional["LoopchainBlock"] = None main_preps: Optional['NodeContainer'] = None next_main_preps: Optional["NodeContainer"] = None if start_height > 0: prev_block_dict = self._block_reader.get_block_by_block_height( start_height - 1) prev_loopchain_block = LoopchainBlock.from_dict(prev_block_dict) # init main_preps preps: list = self._block_reader.load_main_preps(prev_block_dict) main_preps: Optional['NodeContainer'] = NodeContainer.from_list( preps=preps) # when sync from the first block of term, have to initialize next_main_preps here # in that case, invoke_result[3] will be None on first block and can not update next_main_preps block_dict = self._block_reader.get_block_by_block_height( start_height) loopchain_block = LoopchainBlock.from_dict(block_dict) block = _create_iconservice_block(loopchain_block) if self._check_calculation_block(block): preps: list = self._block_reader.load_main_preps(block_dict) next_main_preps: Optional[ 'NodeContainer'] = NodeContainer.from_list(preps=preps) end_height = start_height + count - 1 for height in range(start_height, start_height + count): block_dict: dict = self._block_reader.get_block_by_block_height( height) if block_dict is None: print(f"last block: {height - 1}") break loopchain_block: 'LoopchainBlock' = LoopchainBlock.from_dict( block_dict) block: 'Block' = _create_iconservice_block(loopchain_block) tx_requests: list = create_transaction_requests(loopchain_block) prev_block_generator: Optional[ "Address"] = prev_loopchain_block.leader if prev_loopchain_block else None prev_block_validators: Optional[ List["Address"]] = _create_block_validators( block_dict, prev_block_generator) prev_block_votes: Optional[List[Tuple[ "Address", int]]] = _create_prev_block_votes(block_dict, prev_block_generator, main_preps) if prev_block is not None and prev_block.hash != block.prev_hash: raise Exception() invoke_result = self._engine.invoke( block, tx_requests, prev_block_generator, prev_block_validators, prev_block_votes, ) tx_results, state_root_hash = invoke_result[0], invoke_result[1] main_preps_as_dict: Optional[Dict] = invoke_result[3] commit_state: bytes = self._block_reader.get_commit_state( block_dict, channel) # "commit_state" is the field name of state_root_hash in loopchain block if (height - start_height) % print_block_height == 0: print( f"{height} | {commit_state.hex()[:6]} | {state_root_hash.hex()[:6]} | {len(tx_requests)}" ) if write_precommit_data: self._print_precommit_data(block) try: if stop_on_error: if commit_state: if commit_state != state_root_hash: raise Exception() if height > 0 and not self._check_invoke_result( tx_results): raise Exception() except Exception as e: logging.exception(e) self._print_precommit_data(block) ret: int = 1 break is_calculation_block = self._check_calculation_block(block) if is_calculation_block: word_detector.start() time.sleep(0.5) if iiss_db_backup_path is not None: self._backup_iiss_db(iiss_db_backup_path, block.height) # If no_commit is set to True, the config only affects to the last block to commit if height < end_height: while word_detector.get_hold(): time.sleep(0.5) # Call IconServiceEngine.commit() with a block if not no_commit: self._commit(block) while word_detector.get_hold(): time.sleep(0.5) # Prepare the next iteration self._backup_state_db(block, backup_period) prev_block = block prev_loopchain_block = loopchain_block if next_main_preps: main_preps = next_main_preps next_main_preps = None if main_preps_as_dict is not None: next_main_preps = NodeContainer.from_dict(main_preps_as_dict) self._block_reader.close() word_detector.stop() Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}") return ret def _commit(self, block: "Block"): if "block" in inspect.signature(self._engine.commit).parameters: self._engine.commit(block) else: self._engine.commit(block.height, block.hash, block.hash) def _backup_iiss_db(self, iiss_db_backup_path: Optional[str], block_height: int): iiss_db_path: str = os.path.join(self._engine._state_db_root_path, "iiss") with os.scandir(iiss_db_path) as it: for entry in it: if entry.is_dir( ) and entry.name == Storage.CURRENT_IISS_DB_NAME: dst_path: str = os.path.join( iiss_db_backup_path, f"{Storage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}", ) if os.path.exists(dst_path): shutil.rmtree(dst_path) shutil.copytree(entry.path, dst_path) break def _check_invoke_result(self, tx_results: list): """Compare the transaction results from IconServiceEngine with the results stored in loopchain db If transaction result is not compatible to protocol v3, pass it :param tx_results: the transaction results that IconServiceEngine.invoke() returns :return: True(same) False(different) """ for tx_result in tx_results: tx_info_in_db: dict = self._block_reader.get_transaction_result_by_hash( tx_result.tx_hash.hex()) tx_result_in_db = tx_info_in_db["result"] # tx_v2 dose not have transaction result_v3 if "status" not in tx_result_in_db: continue # information extracted from db status: int = int(tx_result_in_db["status"], 16) tx_hash: bytes = bytes.fromhex(tx_result_in_db["txHash"]) step_used: int = int(tx_result_in_db["stepUsed"], 16) step_price: int = int(tx_result_in_db["stepPrice"], 16) event_logs: list = tx_result_in_db["eventLogs"] step: int = step_used * step_price if tx_hash != tx_result.tx_hash: print(f"tx_hash: {tx_hash.hex()} != {tx_result.tx_hash.hex()}") return False if status != tx_result.status: print(f"status: {status} != {tx_result.status}") return False if step_used != tx_result.step_used: print(f"step_used: {step_used} != {tx_result.step_used}") return False tx_result_step: int = tx_result.step_used * tx_result.step_price if step != tx_result_step: print(f"step: {step} != {tx_result_step}") return False if step_price != tx_result.step_price: print(f"step_price: {step_price} != {tx_result.step_price}") return False if not self._check_event_logs(event_logs, tx_result.event_logs): return False return True @staticmethod def _check_event_logs(event_logs_in_db: list, event_logs_in_tx_result: list): if event_logs_in_db is None: event_logs_in_db = [] if event_logs_in_tx_result is None: event_logs_in_tx_result = [] if len(event_logs_in_db) != len(event_logs_in_tx_result): return False for event_log, _tx_result_event_log in zip(event_logs_in_db, event_logs_in_tx_result): tx_result_event_log: dict = _tx_result_event_log.to_dict() # convert Address to str if "score_address" in tx_result_event_log: score_address: "Address" = tx_result_event_log["score_address"] del tx_result_event_log["score_address"] tx_result_event_log["scoreAddress"] = str(score_address) # convert Address objects to str objects in 'indexes' indexed: list = tx_result_event_log["indexed"] for i in range(len(indexed)): value = indexed[i] indexed[i] = object_to_str(value) data: list = tx_result_event_log["data"] for i in range(len(data)): value = data[i] data[i] = object_to_str(value) if event_log != tx_result_event_log: print(f"{event_log} != {tx_result_event_log}") return False return True def _print_precommit_data(self, block: "Block"): """Print the latest updated states stored in IconServiceEngine :return: """ precommit_data_manager: PrecommitDataManager = getattr( self._engine, "_precommit_data_manager") precommit_data: PrecommitData = precommit_data_manager.get(block.hash) block_batch: BlockBatch = precommit_data.block_batch state_root_hash: bytes = block_batch.digest() filename = f"{block.height}-precommit-data.txt" with open(filename, "wt") as f: for i, key in enumerate(block_batch): value: "TransactionBatchValue" = block_batch[key] if value: hex_value = value.value.hex( ) if value.value is not None else None include_state_root_hash = value.include_state_root_hash line = f"{i}: {key.hex()} - {hex_value}, {include_state_root_hash}" else: line = f"{i}: {key.hex()} - None" print(line) f.write(f"{line}\n") f.write(f"state_root_hash: {state_root_hash.hex()}\n") def _check_calculation_block(self, block: "Block") -> bool: """check calculation block""" context = IconScoreContext(IconScoreContextType.DIRECT) revision = self._engine._get_revision_from_rc(context) context.block = block if revision < Revision.IISS.value: return False start_block = context.engine.iiss.get_start_block_of_calc(context) return start_block == block.height @staticmethod def _backup_state_db(block: "Block", backup_period: int): if backup_period <= 0: return if block.height == 0: return if block.height % backup_period == 0: print(f"----------- Backup statedb: {block.height} ------------") dirname: str = f"block-{'%09d' % block.height}" for basename in (".score", ".statedb"): try: shutil.copytree(basename, f"{dirname}/{basename}/") except FileExistsError: pass def close(self): pass