class IconServiceSyncer(object):
_TAG = "SYNC"
def __init__(self):
self._block_reader = BinBlockDatabaseReader()
self._engine = IconServiceEngine()
self._timer = Timer()
def open(
self,
config_path: str,
fee: bool = True,
audit: bool = True,
deployer_whitelist: bool = False,
score_package_validator: bool = False,
builtin_score_owner: str = "",
):
conf = IconConfig("", default_icon_config)
if config_path != "":
conf.load(config_path)
conf.update_conf({
"builtinScoreOwner": builtin_score_owner,
"service": {
"fee": fee,
"audit": audit,
"scorePackageValidator": score_package_validator,
"deployerWhiteList": deployer_whitelist,
},
})
Logger.load_config(conf)
self._engine.open(conf)
def run(self, *args, **kwargs) -> int:
Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}")
loop = asyncio.get_event_loop()
future = asyncio.Future()
try:
asyncio.ensure_future(
self._wait_for_complete(future, *args, **kwargs))
loop.run_until_complete(future)
finally:
for task in asyncio.Task.all_tasks():
task.cancel()
loop.close()
ret = future.result()
Logger.debug(tag=self._TAG, msg=f"run() end: {ret}")
return ret
async def _wait_for_complete(self, result_future: asyncio.Future, *args,
**kwargs):
Logger.debug(tag=self._TAG, msg="_wait_for_complete() start")
# Wait for rc to be ready
future = self._engine.get_ready_future()
await future
with ThreadPoolExecutor(max_workers=1) as executor:
# Call IconServiceEngine.hello()
f = executor.submit(self._hello)
future = asyncio.wrap_future(f)
await future
# Start to sync blocks
f = executor.submit(self._run, *args, **kwargs)
f = asyncio.wrap_future(f)
for fut in asyncio.as_completed([f]):
try:
ret = await fut
except Exception as exc:
self._engine.close()
# Wait to stop ipc_server for 1s
await asyncio.sleep(1)
result_future.set_exception(exc)
else:
self._engine.close()
# Wait to stop ipc_server for 1s
await asyncio.sleep(1)
Logger.debug(tag=self._TAG,
msg="_wait_for_complete() end1")
result_future.set_result(ret)
Logger.debug(tag=self._TAG,
msg="_wait_for_complete() end2")
def _hello(self):
self._engine.hello()
def _run(
self,
db_path: str,
channel: str,
start_height: int = 0,
count: int = 99_999_999,
stop_on_error: bool = True,
no_commit: bool = False,
backup_period: int = 0,
write_precommit_data: bool = False,
print_block_height: int = 1,
iiss_db_backup_path: Optional[str] = None,
) -> int:
"""Begin to synchronize IconServiceEngine with blocks from loopchain db
:param db_path: loopchain db path
:param channel: channel name used as a key to get commit_state in loopchain db
:param start_height: start height to sync
:param count: The number of blocks to sync
:param stop_on_error: If error happens, stop syncing
:param no_commit: Do not commit
:param backup_period: state backup period in block
:param write_precommit_data:
:param print_block_height: print every this block height
:return: 0(success), otherwise(error)
"""
Logger.debug(tag=self._TAG, msg="_run() start")
word_detector = WordDetector(
filename="iconservice.log",
block_word=r"CALCULATE\(",
release_word=r"CALCULATE_DONE\(",
)
ret: int = 0
self._block_reader.open(db_path)
print("block_height | commit_state | state_root_hash | tx_count")
prev_bin_block: Optional["BinBlock"] = None
prev_block: Optional["Block"] = None
main_preps: Optional['NodeContainer'] = None
next_main_preps: Optional["NodeContainer"] = None
if start_height > 0:
prev_bin_block: Optional[
BinBlock] = self._block_reader.get_block_by_height(
start_height - 1)
# init main_preps
preps: list = self._block_reader.load_main_preps(
reps_hash=prev_bin_block.reps_hash)
main_preps: Optional['NodeContainer'] = NodeContainer.from_list(
preps=preps)
# when sync from the first block of term, have to initialize next_main_preps here
# in that case, invoke_result[3] will be None on first block and can not update next_main_preps
cur_bin_block: Optional[
BinBlock] = self._block_reader.get_block_by_height(
start_height)
block: "Block" = create_iconservice_block(cur_bin_block)
if self._check_calculation_block(block):
preps: list = self._block_reader.load_main_preps(
reps_hash=cur_bin_block.reps_hash)
next_main_preps: Optional[
'NodeContainer'] = NodeContainer.from_list(preps=preps)
end_height = start_height + count - 1
self._timer.start()
for height in range(start_height, start_height + count):
bin_block = self._block_reader.get_block_by_height(height)
if bin_block is None:
print(f"last block: {height - 1}")
break
block: "Block" = create_iconservice_block(bin_block)
tx_requests: List[Dict[str, Any]] = create_transaction_requests(
bin_block.transactions)
prev_block_generator: Optional[
"Address"] = prev_bin_block.leader if prev_bin_block else None
prev_block_validators: Optional[
List["Address"]] = create_block_validators(
bin_block.prev_votes, prev_block_generator)
prev_block_votes: Optional[List[
Tuple["Address",
int]]] = create_prev_block_votes(bin_block.prev_votes,
prev_block_generator,
main_preps)
Logger.info(tag=self._TAG,
msg=f"prev_block_generator={prev_block_generator}")
Logger.info(tag=self._TAG,
msg=f"prev_block_validators={prev_block_validators}")
Logger.info(tag=self._TAG,
msg=f"prev_block_votes={prev_block_votes}")
if prev_block is not None and prev_block.hash != block.prev_hash:
raise Exception(f"Invalid prev_block_hash: height={height}")
invoke_result = self._engine.invoke(
block,
tx_requests,
prev_block_generator,
prev_block_validators,
prev_block_votes,
)
tx_results, state_root_hash = invoke_result[0], invoke_result[1]
main_preps_as_dict: Optional[Dict] = invoke_result[3]
commit_state: bytes = bin_block.state_hash
# "commit_state" is the field name of state_root_hash in loopchain block
if (height - start_height) % print_block_height == 0:
self._print_status(height, start_height, count, commit_state,
state_root_hash, len(tx_requests))
if write_precommit_data:
self._print_precommit_data(block)
try:
if stop_on_error:
if commit_state:
if commit_state != state_root_hash:
raise Exception("state_root_hash mismatch")
if height > 0 and not self._check_invoke_result(
tx_results):
raise Exception("tx_result mismatch")
except Exception as e:
logging.exception(e)
self._print_precommit_data(block)
ret: int = 1
break
is_calculation_block = self._check_calculation_block(block)
if is_calculation_block:
word_detector.start()
time.sleep(0.5)
if iiss_db_backup_path is not None:
self._backup_iiss_db(iiss_db_backup_path, block.height)
# If no_commit is set to True, the config only affects to the last block to commit
if not no_commit or height < end_height:
while word_detector.get_hold():
time.sleep(0.5)
# Call IconServiceEngine.commit() with a block
self._commit(block)
while word_detector.get_hold():
time.sleep(0.5)
# Prepare the next iteration
self._backup_state_db(block, backup_period)
prev_block: 'Block' = block
prev_bin_block: 'BinBlock' = bin_block
if next_main_preps:
main_preps = next_main_preps
next_main_preps = None
if main_preps_as_dict is not None:
next_main_preps = NodeContainer.from_dict(main_preps_as_dict)
self._block_reader.close()
word_detector.stop()
Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}")
return ret
class IconScoreInnerTask(object):
def __init__(self, conf: dict):
self._conf = conf
self._thread_flag = ENABLE_THREAD_FLAG
self._icon_service_engine = IconServiceEngine()
self._open()
self._thread_pool = {
THREAD_INVOKE: ThreadPoolExecutor(1),
THREAD_STATUS: ThreadPoolExecutor(1),
THREAD_QUERY: ThreadPoolExecutor(1),
THREAD_ESTIMATE: ThreadPoolExecutor(1),
THREAD_VALIDATE: ThreadPoolExecutor(1)
}
def _open(self):
Logger.info(tag=_TAG, msg="_open() start")
self._icon_service_engine.open(self._conf)
Logger.info(tag=_TAG, msg="_open() end")
def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool:
return (self._thread_flag & flag) == flag
def _check_icon_service_ready(self):
if not self._icon_service_engine.is_reward_calculator_ready():
raise ServiceNotReadyException("Reward Calculator is not ready")
@staticmethod
def _log_exception(e: BaseException, tag: str) -> None:
Logger.exception(str(e), tag)
Logger.error(str(e), tag)
@message_queue_task
async def hello(self):
Logger.info(tag=_TAG, msg='hello() start')
ready_future = self._icon_service_engine.get_ready_future()
await ready_future
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello)
else:
ret = self._hello()
Logger.info(tag=_TAG, msg='hello() end')
return ret
def _hello(self):
return self._icon_service_engine.hello()
def cleanup(self):
Logger.info(tag=_TAG, msg="cleanup() start")
# shutdown thread pool executors
for executor in self._thread_pool.values():
executor.shutdown()
# close ICON Service
if self._icon_service_engine:
self._icon_service_engine.close()
self._icon_service_engine = None
Logger.info(tag=_TAG, msg="cleanup() end")
@message_queue_task
async def close(self):
Logger.info(tag=_TAG, msg="close() stop event loop")
self._close()
@staticmethod
def _close():
asyncio.get_event_loop().stop()
@message_queue_task
async def invoke(self, request: dict) -> dict:
Logger.debug(tag=_TAG, msg=f'invoke() start')
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
ret: dict = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._invoke, request)
else:
ret: dict = self._invoke(request)
Logger.debug(tag=_TAG, msg=f'invoke() end')
return ret
def _invoke(self, request: dict):
"""Process transactions in a block
:param request:
:return:
"""
Logger.info(tag=_TAG, msg=f'INVOKE Request: {request}')
try:
params = TypeConverter.convert(request, ParamType.INVOKE)
converted_block_params = params['block']
block = Block.from_dict(converted_block_params)
Logger.info(tag=_TAG, msg=f'INVOKE: BH={block.height}')
converted_tx_requests = params['transactions']
convert_tx_result_to_dict: bool = 'isBlockEditable' in params
converted_is_block_editable = params.get('isBlockEditable', False)
converted_prev_block_generator = params.get('prevBlockGenerator')
converted_prev_block_validators = params.get('prevBlockValidators')
converted_prev_votes = params.get('prevBlockVotes')
tx_results, state_root_hash, added_transactions, next_preps, is_shutdown = \
self._icon_service_engine.invoke(
block=block,
tx_requests=converted_tx_requests,
prev_block_generator=converted_prev_block_generator,
prev_block_validators=converted_prev_block_validators,
prev_block_votes=converted_prev_votes,
is_block_editable=converted_is_block_editable
)
if convert_tx_result_to_dict:
convert_tx_results = [tx_result.to_dict(to_camel_case) for tx_result in tx_results]
else:
# old version
convert_tx_results = {bytes.hex(tx_result.tx_hash): tx_result.to_dict(to_camel_case)
for tx_result in tx_results}
results = {
'txResults': convert_tx_results,
'stateRootHash': bytes.hex(state_root_hash),
'addedTransactions': added_transactions
}
if next_preps:
results["prep"] = next_preps
if is_shutdown:
results["is_shutdown"] = True
response = MakeResponse.make_response(results)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except InvalidBaseTransactionException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
finally:
if self._icon_service_engine:
self._icon_service_engine.clear_context_stack()
Logger.info(tag=_TAG, msg=f'INVOKE Response: {json.dumps(response, cls=BytesToHexJSONEncoder)}')
return response
@message_queue_task
async def query(self, request: dict) -> dict:
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
return await self._get_query_response(request)
async def _get_query_response(self, request: dict) -> dict:
try:
value = await self._execute_query(request)
if isinstance(value, Address):
value = str(value)
response = MakeResponse.make_response(value)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
self._icon_service_engine.clear_context_stack()
return response
async def _execute_query(self, request: dict):
method_name: str = request['method']
if method_name == RPCMethod.DEBUG_ESTIMATE_STEP:
method: callable = self._estimate
args = [request]
else:
method: callable = self._query
args = [request, method_name]
if self._is_thread_flag_on(EnableThreadFlag.QUERY):
return await asyncio.get_event_loop(). \
run_in_executor(self._thread_pool[QUERY_THREAD_MAPPER[method_name]],
method, *args)
else:
return method(*args)
def _estimate(self, request: dict):
converted_request = TypeConverter.convert(request, ParamType.INVOKE_TRANSACTION)
return self._icon_service_engine.estimate_step(converted_request)
def _query(self, request: dict, method: str):
converted_request = TypeConverter.convert(request, ParamType.QUERY)
return self._icon_service_engine.query(method, converted_request['params'])
@message_queue_task
async def call(self, request: dict):
Logger.info(tag=_TAG, msg=f'call() start: {request}')
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
if self._is_thread_flag_on(EnableThreadFlag.QUERY):
loop = asyncio.get_event_loop()
ret = await loop.run_in_executor(self._thread_pool[THREAD_QUERY],
self._call, request)
else:
ret = self._call(request)
Logger.info(tag=_TAG, msg=f'call() end: {ret}')
return ret
def _call(self, request: dict):
try:
response = self._icon_service_engine.inner_call(request)
if isinstance(response, Address):
response = str(response)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
return response
@message_queue_task
async def write_precommit_state(self, request: dict):
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._write_precommit_state, request)
else:
ret = self._write_precommit_state(request)
return ret
def _write_precommit_state(self, request: dict) -> dict:
Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Request: {request}')
try:
converted_params = TypeConverter.convert(request, ParamType.WRITE_PRECOMMIT)
block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT]
instant_block_hash: bytes = converted_params[ConstantKeys.OLD_BLOCK_HASH]
block_hash = converted_params[ConstantKeys.NEW_BLOCK_HASH]
Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE: '
f'BH={block_height} '
f'instant_block_hash={bytes_to_hex(instant_block_hash)} '
f'block_hash={bytes_to_hex(block_hash)}')
self._icon_service_engine.commit(block_height, instant_block_hash, block_hash)
response = MakeResponse.make_response(ExceptionCode.OK)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
Logger.info(tag=_TAG, msg=f'WRITE_PRECOMMIT_STATE Response: {response}')
return response
@message_queue_task
async def remove_precommit_state(self, request: dict):
Logger.info(tag=_TAG, msg=f'remove_precommit_state() start')
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
"""
Unused API
"""
return {}
@message_queue_task
async def rollback(self, request: dict):
"""Go back to the state of the given previous block
:param request:
:return:
"""
Logger.info(tag=_TAG, msg=f"rollback() start")
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
response = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._rollback, request)
else:
response = self._rollback(request)
Logger.info(tag=_TAG, msg=f"rollback() end")
return response
def _rollback(self, request: dict) -> dict:
Logger.info(tag=_TAG, msg=f"ROLLBACK Request: {request}")
try:
converted_params = TypeConverter.convert(request, ParamType.ROLLBACK)
block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT]
block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH]
Logger.info(tag=_TAG, msg=f"ROLLBACK: BH={block_height} block_hash={bytes_to_hex(block_hash)}")
response: dict = self._icon_service_engine.rollback(block_height, block_hash)
response = MakeResponse.make_response(response)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except BaseException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
Logger.info(tag=_TAG, msg=f"ROLLBACK Response: {response}")
return response
@message_queue_task
async def validate_transaction(self, request: dict):
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
if self._is_thread_flag_on(EnableThreadFlag.VALIDATE):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_VALIDATE],
self._validate_transaction, request)
else:
return self._validate_transaction(request)
def _validate_transaction(self, request: dict):
try:
Logger.info(tag=_TAG, msg=f'validate_transaction Request: {request}')
converted_request = TypeConverter.convert(request, ParamType.VALIDATE_TRANSACTION)
self._icon_service_engine.validate_transaction(converted_request, request)
response = MakeResponse.make_response(ExceptionCode.OK)
except FatalException as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, _TAG)
response = MakeResponse.make_error_response(icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
self._icon_service_engine.clear_context_stack()
return response
@message_queue_task
async def change_block_hash(self, _params):
try:
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
return ExceptionCode.OK
@message_queue_task
async def dos_guard(self, params: dict) -> dict:
try:
Logger.info(tag=_TAG, msg=f'dos_guard: params: {params}')
self._check_icon_service_ready()
except ServiceNotReadyException as e:
return MakeResponse.make_error_response(e.code, str(e))
try:
self._icon_service_engine.dos_guard.run(_from=params["from"])
response = MakeResponse.make_response(ExceptionCode.OK)
except Exception as e:
self._log_exception(e, _TAG)
response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e))
return response
class IconScoreInnerTask(object):
def __init__(self, conf: 'IconConfig'):
self._conf = conf
self._thread_flag = ENABLE_THREAD_FLAG
self._icon_service_engine = IconServiceEngine()
self._open()
self._thread_pool = {
THREAD_INVOKE: ThreadPoolExecutor(1),
THREAD_QUERY: ThreadPoolExecutor(1),
THREAD_VALIDATE: ThreadPoolExecutor(1)
}
def _open(self):
Logger.info("icon_score_service open", ICON_INNER_LOG_TAG)
self._icon_service_engine.open(self._conf)
def _is_thread_flag_on(self, flag: 'EnableThreadFlag') -> bool:
return (self._thread_flag & flag) == flag
def _check_icon_service_ready(self):
if not self._icon_service_engine.is_reward_calculator_ready():
raise ServiceNotReadyException("Reward Calculator is not ready")
@staticmethod
def _log_exception(e: BaseException,
tag: str = ICON_INNER_LOG_TAG) -> None:
Logger.exception(str(e), tag)
Logger.error(str(e), tag)
@message_queue_task
async def hello(self):
Logger.info('hello() start', ICON_INNER_LOG_TAG)
ready_future = self._icon_service_engine.get_ready_future()
await ready_future
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._hello)
else:
ret = self._hello()
Logger.info('hello() end', ICON_INNER_LOG_TAG)
return ret
def _hello(self):
return self._icon_service_engine.hello()
def _close(self):
Logger.info(tag=_TAG, msg="_close() start")
if self._icon_service_engine:
self._icon_service_engine.close()
self._icon_service_engine = None
MessageQueueService.loop.stop()
Logger.info(tag=_TAG, msg="_close() end")
@message_queue_task
async def close(self):
Logger.info(tag=_TAG, msg="close() start")
self._close()
Logger.info(tag=_TAG, msg="close() end")
@message_queue_task
async def invoke(self, request: dict):
Logger.info(f'invoke request with {request}', ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._invoke, request)
else:
return self._invoke(request)
def _invoke(self, request: dict):
"""Process transactions in a block
:param request:
:return:
"""
response = None
try:
params = TypeConverter.convert(request, ParamType.INVOKE)
converted_block_params = params['block']
block = Block.from_dict(converted_block_params)
converted_tx_requests = params['transactions']
convert_tx_result_to_dict: bool = 'isBlockEditable' in params
converted_is_block_editable = params.get('isBlockEditable', False)
converted_prev_block_generator = params.get('prevBlockGenerator')
converted_prev_block_validators = params.get('prevBlockValidators')
converted_prev_votes = params.get('prevBlockVotes')
tx_results, state_root_hash, added_transactions, main_prep_as_dict = self._icon_service_engine.invoke(
block=block,
tx_requests=converted_tx_requests,
prev_block_generator=converted_prev_block_generator,
prev_block_validators=converted_prev_block_validators,
prev_block_votes=converted_prev_votes,
is_block_editable=converted_is_block_editable)
if convert_tx_result_to_dict:
convert_tx_results = [
tx_result.to_dict(to_camel_case)
for tx_result in tx_results
]
else:
# old version
convert_tx_results = {
bytes.hex(tx_result.tx_hash):
tx_result.to_dict(to_camel_case)
for tx_result in tx_results
}
results = {
'txResults': convert_tx_results,
'stateRootHash': bytes.hex(state_root_hash),
'addedTransactions': added_transactions
}
if main_prep_as_dict:
results["prep"] = main_prep_as_dict
Logger.info(f'invoke origin response with {results}',
ICON_INNER_LOG_TAG)
response = MakeResponse.make_response(results)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except InvalidBaseTransactionException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
if self._icon_service_engine:
self._icon_service_engine.clear_context_stack()
return response
@message_queue_task
async def query(self, request: dict):
Logger.debug(f'query request with {request}', ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.QUERY):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_QUERY],
self._query, request)
else:
return self._query(request)
def _query(self, request: dict):
response = None
try:
method = request['method']
if method == 'debug_estimateStep':
converted_request = TypeConverter.convert(
request, ParamType.INVOKE_TRANSACTION)
value = self._icon_service_engine.estimate_step(
converted_request)
else:
converted_request = TypeConverter.convert(
request, ParamType.QUERY)
value = self._icon_service_engine.query(
method, converted_request['params'])
if isinstance(value, Address):
value = str(value)
response = MakeResponse.make_response(value)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.debug(f'query response with {response}', ICON_INNER_LOG_TAG)
self._icon_service_engine.clear_context_stack()
return response
@message_queue_task
async def call(self, request: dict):
Logger.info(f'call request with {request}', ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.QUERY):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_QUERY],
self._call, request)
else:
return self._call(request)
def _call(self, request: dict):
response = None
try:
response = self._icon_service_engine.inner_call(request)
if isinstance(response, Address):
response = str(response)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.info(f'call response with {response}', ICON_INNER_LOG_TAG)
return response
@message_queue_task
async def write_precommit_state(self, request: dict):
Logger.info(f'write_precommit_state request with {request}',
ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._write_precommit_state,
request)
else:
return self._write_precommit_state(request)
@staticmethod
def _get_block_info_for_precommit_state(
converted_block_params: dict
) -> Tuple[int, bytes, Optional[bytes]]:
block_height: int = converted_block_params[ConstantKeys.BLOCK_HEIGHT]
block_hash: Optional[bytes] = None
if ConstantKeys.BLOCK_HASH in converted_block_params:
instant_block_hash: bytes = converted_block_params[
ConstantKeys.BLOCK_HASH]
else:
instant_block_hash: bytes = converted_block_params[
ConstantKeys.OLD_BLOCK_HASH]
block_hash = converted_block_params[ConstantKeys.NEW_BLOCK_HASH]
return block_height, instant_block_hash, block_hash
def _write_precommit_state(self, request: dict):
response = None
try:
converted_block_params = TypeConverter.convert(
request, ParamType.WRITE_PRECOMMIT)
block_height, instant_block_hash, block_hash = \
self._get_block_info_for_precommit_state(converted_block_params)
self._icon_service_engine.commit(block_height, instant_block_hash,
block_hash)
response = MakeResponse.make_response(ExceptionCode.OK)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.info(f'write_precommit_state response with {response}',
ICON_INNER_LOG_TAG)
return response
@message_queue_task
async def remove_precommit_state(self, request: dict):
Logger.info(f'remove_precommit_state request with {request}',
ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
self._remove_precommit_state,
request)
else:
return self._remove_precommit_state(request)
def _remove_precommit_state(self, request: dict):
response = None
try:
converted_block_params = TypeConverter.convert(
request, ParamType.WRITE_PRECOMMIT)
block_height, instant_block_hash, _ = \
self._get_block_info_for_precommit_state(converted_block_params)
self._icon_service_engine.remove_precommit_state(
block_height, instant_block_hash)
response = MakeResponse.make_response(ExceptionCode.OK)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.info(f'remove_precommit_state response with {response}',
ICON_INNER_LOG_TAG)
return response
@message_queue_task
async def rollback(self, request: dict):
"""Go back to the state of the given previous block
:param request:
:return:
"""
Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() start: {request}")
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
loop = asyncio.get_event_loop()
response = await loop.run_in_executor(
self._thread_pool[THREAD_INVOKE], self._rollback, request)
else:
response = self._rollback(request)
Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() end: {response}")
return response
def _rollback(self, request: dict) -> dict:
Logger.info(tag=ICON_INNER_LOG_TAG,
msg=f"_rollback() start: {request}")
response = {}
try:
converted_params = TypeConverter.convert(request,
ParamType.ROLLBACK)
block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT]
block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH]
response: dict = self._icon_service_engine.rollback(
block_height, block_hash)
response = MakeResponse.make_response(response)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
self._close()
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except BaseException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.info(tag=ICON_INNER_LOG_TAG,
msg=f"_rollback() end: {response}")
return response
@message_queue_task
async def validate_transaction(self, request: dict):
Logger.debug(f'pre_validate_check request with {request}',
ICON_INNER_LOG_TAG)
self._check_icon_service_ready()
if self._is_thread_flag_on(EnableThreadFlag.VALIDATE):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(
self._thread_pool[THREAD_VALIDATE], self._validate_transaction,
request)
else:
return self._validate_transaction(request)
def _validate_transaction(self, request: dict):
response = None
try:
converted_request = TypeConverter.convert(
request, ParamType.VALIDATE_TRANSACTION)
self._icon_service_engine.validate_transaction(converted_request)
response = MakeResponse.make_response(ExceptionCode.OK)
except FatalException as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
except IconServiceBaseException as icon_e:
self._log_exception(icon_e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
icon_e.code, icon_e.message)
except Exception as e:
self._log_exception(e, ICON_SERVICE_LOG_TAG)
response = MakeResponse.make_error_response(
ExceptionCode.SYSTEM_ERROR, str(e))
finally:
Logger.debug(f'pre_validate_check response with {response}',
ICON_INNER_LOG_TAG)
self._icon_service_engine.clear_context_stack()
return response
@message_queue_task
async def change_block_hash(self, _params):
self._check_icon_service_ready()
return ExceptionCode.OK
class IconServiceSyncer(object):
_TAG = "SYNC"
def __init__(self):
self._block_reader = BlockDatabaseReader()
self._engine = IconServiceEngine()
def open(
self,
config_path: str,
fee: bool = True,
audit: bool = True,
deployer_whitelist: bool = False,
score_package_validator: bool = False,
builtin_score_owner: str = "",
):
conf = IconConfig("", default_icon_config)
if config_path != "":
conf.load(config_path)
conf.update_conf({
"builtinScoreOwner": builtin_score_owner,
"service": {
"fee": fee,
"audit": audit,
"scorePackageValidator": score_package_validator,
"deployerWhiteList": deployer_whitelist,
},
})
Logger.load_config(conf)
self._engine.open(conf)
def run(self, *args, **kwargs) -> int:
Logger.debug(tag=self._TAG, msg=f"run() start: {args} {kwargs}")
loop = asyncio.get_event_loop()
future = asyncio.Future()
try:
asyncio.ensure_future(
self._wait_for_complete(future, *args, **kwargs))
loop.run_until_complete(future)
finally:
for task in asyncio.Task.all_tasks():
task.cancel()
loop.close()
ret = future.result()
Logger.debug(tag=self._TAG, msg=f"run() end: {ret}")
return ret
async def _wait_for_complete(self, result_future: asyncio.Future, *args,
**kwargs):
Logger.debug(tag=self._TAG, msg="_wait_for_complete() start")
# Wait for rc to be ready
future = self._engine.get_ready_future()
await future
with ThreadPoolExecutor(max_workers=1) as executor:
# Call IconServiceEngine.hello()
f = executor.submit(self._hello)
future = asyncio.wrap_future(f)
await future
# Start to sync blocks
f = executor.submit(self._run, *args, **kwargs)
f = asyncio.wrap_future(f)
for fut in asyncio.as_completed([f]):
try:
ret = await fut
except Exception as exc:
self._engine.close()
# Wait to stop ipc_server for 1s
await asyncio.sleep(1)
result_future.set_exception(exc)
else:
self._engine.close()
# Wait to stop ipc_server for 1s
await asyncio.sleep(1)
Logger.debug(tag=self._TAG,
msg="_wait_for_complete() end1")
result_future.set_result(ret)
Logger.debug(tag=self._TAG,
msg="_wait_for_complete() end2")
def _hello(self):
self._engine.hello()
def _run(
self,
db_path: str,
channel: str,
start_height: int = 0,
count: int = 99999999,
stop_on_error: bool = True,
no_commit: bool = False,
backup_period: int = 0,
write_precommit_data: bool = False,
print_block_height: int = 1,
iiss_db_backup_path: Optional[str] = None,
) -> int:
"""Begin to synchronize IconServiceEngine with blocks from loopchain db
:param db_path: loopchain db path
:param channel: channel name used as a key to get commit_state in loopchain db
:param start_height: start height to sync
:param count: The number of blocks to sync
:param stop_on_error: If error happens, stop syncing
:param no_commit: Do not commit
:param backup_period: state backup period in block
:param write_precommit_data:
:param print_block_height: print every this block height
:return: 0(success), otherwise(error)
"""
Logger.debug(tag=self._TAG, msg="_run() start")
word_detector = WordDetector(
filename="iconservice.log",
block_word=r"CALCULATE\(",
release_word=r"CALCULATE_DONE\(",
)
ret: int = 0
self._block_reader.open(db_path)
print("block_height | commit_state | state_root_hash | tx_count")
prev_block: Optional["Block"] = None
prev_loopchain_block: Optional["LoopchainBlock"] = None
main_preps: Optional['NodeContainer'] = None
next_main_preps: Optional["NodeContainer"] = None
if start_height > 0:
prev_block_dict = self._block_reader.get_block_by_block_height(
start_height - 1)
prev_loopchain_block = LoopchainBlock.from_dict(prev_block_dict)
# init main_preps
preps: list = self._block_reader.load_main_preps(prev_block_dict)
main_preps: Optional['NodeContainer'] = NodeContainer.from_list(
preps=preps)
# when sync from the first block of term, have to initialize next_main_preps here
# in that case, invoke_result[3] will be None on first block and can not update next_main_preps
block_dict = self._block_reader.get_block_by_block_height(
start_height)
loopchain_block = LoopchainBlock.from_dict(block_dict)
block = _create_iconservice_block(loopchain_block)
if self._check_calculation_block(block):
preps: list = self._block_reader.load_main_preps(block_dict)
next_main_preps: Optional[
'NodeContainer'] = NodeContainer.from_list(preps=preps)
end_height = start_height + count - 1
for height in range(start_height, start_height + count):
block_dict: dict = self._block_reader.get_block_by_block_height(
height)
if block_dict is None:
print(f"last block: {height - 1}")
break
loopchain_block: 'LoopchainBlock' = LoopchainBlock.from_dict(
block_dict)
block: 'Block' = _create_iconservice_block(loopchain_block)
tx_requests: list = create_transaction_requests(loopchain_block)
prev_block_generator: Optional[
"Address"] = prev_loopchain_block.leader if prev_loopchain_block else None
prev_block_validators: Optional[
List["Address"]] = _create_block_validators(
block_dict, prev_block_generator)
prev_block_votes: Optional[List[Tuple[
"Address",
int]]] = _create_prev_block_votes(block_dict,
prev_block_generator,
main_preps)
if prev_block is not None and prev_block.hash != block.prev_hash:
raise Exception()
invoke_result = self._engine.invoke(
block,
tx_requests,
prev_block_generator,
prev_block_validators,
prev_block_votes,
)
tx_results, state_root_hash = invoke_result[0], invoke_result[1]
main_preps_as_dict: Optional[Dict] = invoke_result[3]
commit_state: bytes = self._block_reader.get_commit_state(
block_dict, channel)
# "commit_state" is the field name of state_root_hash in loopchain block
if (height - start_height) % print_block_height == 0:
print(
f"{height} | {commit_state.hex()[:6]} | {state_root_hash.hex()[:6]} | {len(tx_requests)}"
)
if write_precommit_data:
self._print_precommit_data(block)
try:
if stop_on_error:
if commit_state:
if commit_state != state_root_hash:
raise Exception()
if height > 0 and not self._check_invoke_result(
tx_results):
raise Exception()
except Exception as e:
logging.exception(e)
self._print_precommit_data(block)
ret: int = 1
break
is_calculation_block = self._check_calculation_block(block)
if is_calculation_block:
word_detector.start()
time.sleep(0.5)
if iiss_db_backup_path is not None:
self._backup_iiss_db(iiss_db_backup_path, block.height)
# If no_commit is set to True, the config only affects to the last block to commit
if height < end_height:
while word_detector.get_hold():
time.sleep(0.5)
# Call IconServiceEngine.commit() with a block
if not no_commit:
self._commit(block)
while word_detector.get_hold():
time.sleep(0.5)
# Prepare the next iteration
self._backup_state_db(block, backup_period)
prev_block = block
prev_loopchain_block = loopchain_block
if next_main_preps:
main_preps = next_main_preps
next_main_preps = None
if main_preps_as_dict is not None:
next_main_preps = NodeContainer.from_dict(main_preps_as_dict)
self._block_reader.close()
word_detector.stop()
Logger.debug(tag=self._TAG, msg=f"_run() end: {ret}")
return ret
def _commit(self, block: "Block"):
if "block" in inspect.signature(self._engine.commit).parameters:
self._engine.commit(block)
else:
self._engine.commit(block.height, block.hash, block.hash)
def _backup_iiss_db(self, iiss_db_backup_path: Optional[str],
block_height: int):
iiss_db_path: str = os.path.join(self._engine._state_db_root_path,
"iiss")
with os.scandir(iiss_db_path) as it:
for entry in it:
if entry.is_dir(
) and entry.name == Storage.CURRENT_IISS_DB_NAME:
dst_path: str = os.path.join(
iiss_db_backup_path,
f"{Storage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}",
)
if os.path.exists(dst_path):
shutil.rmtree(dst_path)
shutil.copytree(entry.path, dst_path)
break
def _check_invoke_result(self, tx_results: list):
"""Compare the transaction results from IconServiceEngine
with the results stored in loopchain db
If transaction result is not compatible to protocol v3, pass it
:param tx_results: the transaction results that IconServiceEngine.invoke() returns
:return: True(same) False(different)
"""
for tx_result in tx_results:
tx_info_in_db: dict = self._block_reader.get_transaction_result_by_hash(
tx_result.tx_hash.hex())
tx_result_in_db = tx_info_in_db["result"]
# tx_v2 dose not have transaction result_v3
if "status" not in tx_result_in_db:
continue
# information extracted from db
status: int = int(tx_result_in_db["status"], 16)
tx_hash: bytes = bytes.fromhex(tx_result_in_db["txHash"])
step_used: int = int(tx_result_in_db["stepUsed"], 16)
step_price: int = int(tx_result_in_db["stepPrice"], 16)
event_logs: list = tx_result_in_db["eventLogs"]
step: int = step_used * step_price
if tx_hash != tx_result.tx_hash:
print(f"tx_hash: {tx_hash.hex()} != {tx_result.tx_hash.hex()}")
return False
if status != tx_result.status:
print(f"status: {status} != {tx_result.status}")
return False
if step_used != tx_result.step_used:
print(f"step_used: {step_used} != {tx_result.step_used}")
return False
tx_result_step: int = tx_result.step_used * tx_result.step_price
if step != tx_result_step:
print(f"step: {step} != {tx_result_step}")
return False
if step_price != tx_result.step_price:
print(f"step_price: {step_price} != {tx_result.step_price}")
return False
if not self._check_event_logs(event_logs, tx_result.event_logs):
return False
return True
@staticmethod
def _check_event_logs(event_logs_in_db: list,
event_logs_in_tx_result: list):
if event_logs_in_db is None:
event_logs_in_db = []
if event_logs_in_tx_result is None:
event_logs_in_tx_result = []
if len(event_logs_in_db) != len(event_logs_in_tx_result):
return False
for event_log, _tx_result_event_log in zip(event_logs_in_db,
event_logs_in_tx_result):
tx_result_event_log: dict = _tx_result_event_log.to_dict()
# convert Address to str
if "score_address" in tx_result_event_log:
score_address: "Address" = tx_result_event_log["score_address"]
del tx_result_event_log["score_address"]
tx_result_event_log["scoreAddress"] = str(score_address)
# convert Address objects to str objects in 'indexes'
indexed: list = tx_result_event_log["indexed"]
for i in range(len(indexed)):
value = indexed[i]
indexed[i] = object_to_str(value)
data: list = tx_result_event_log["data"]
for i in range(len(data)):
value = data[i]
data[i] = object_to_str(value)
if event_log != tx_result_event_log:
print(f"{event_log} != {tx_result_event_log}")
return False
return True
def _print_precommit_data(self, block: "Block"):
"""Print the latest updated states stored in IconServiceEngine
:return:
"""
precommit_data_manager: PrecommitDataManager = getattr(
self._engine, "_precommit_data_manager")
precommit_data: PrecommitData = precommit_data_manager.get(block.hash)
block_batch: BlockBatch = precommit_data.block_batch
state_root_hash: bytes = block_batch.digest()
filename = f"{block.height}-precommit-data.txt"
with open(filename, "wt") as f:
for i, key in enumerate(block_batch):
value: "TransactionBatchValue" = block_batch[key]
if value:
hex_value = value.value.hex(
) if value.value is not None else None
include_state_root_hash = value.include_state_root_hash
line = f"{i}: {key.hex()} - {hex_value}, {include_state_root_hash}"
else:
line = f"{i}: {key.hex()} - None"
print(line)
f.write(f"{line}\n")
f.write(f"state_root_hash: {state_root_hash.hex()}\n")
def _check_calculation_block(self, block: "Block") -> bool:
"""check calculation block"""
context = IconScoreContext(IconScoreContextType.DIRECT)
revision = self._engine._get_revision_from_rc(context)
context.block = block
if revision < Revision.IISS.value:
return False
start_block = context.engine.iiss.get_start_block_of_calc(context)
return start_block == block.height
@staticmethod
def _backup_state_db(block: "Block", backup_period: int):
if backup_period <= 0:
return
if block.height == 0:
return
if block.height % backup_period == 0:
print(f"----------- Backup statedb: {block.height} ------------")
dirname: str = f"block-{'%09d' % block.height}"
for basename in (".score", ".statedb"):
try:
shutil.copytree(basename, f"{dirname}/{basename}/")
except FileExistsError:
pass
def close(self):
pass
