def __init__(self, timer_key, duration, timer_service: TimerService, callback, callback_lock): self.__slot = 0 self.__delayed = True self.__timer_key = timer_key self.__timer_service = timer_service self.__callback = callback self.__callback_lock = callback_lock timer_service.add_timer( timer_key, Timer(target=timer_key, duration=duration, is_repeat=True, is_run_at_start=True, callback=self.__timer_callback))
class _Broadcaster: """broadcast class for each channel""" THREAD_VARIABLE_PEER_STATUS = "peer_status" def __init__(self, channel: str, self_target: str = None): self.__channel = channel self.__self_target = self_target self.__audience = {} # self.__audience[peer_target] = stub_manager self.__thread_variables = dict() self.__thread_variables[ self.THREAD_VARIABLE_PEER_STATUS] = PeerThreadStatus.normal if conf.IS_BROADCAST_ASYNC: self.__broadcast_run = self.__broadcast_run_async else: self.__broadcast_run = self.__broadcast_run_sync self.__handler_map = { BroadcastCommand.CREATE_TX: self.__handler_create_tx, BroadcastCommand.UPDATE_AUDIENCE: self.__handler_update_audience, BroadcastCommand.BROADCAST: self.__handler_broadcast, BroadcastCommand.SEND_TO_SINGLE_TARGET: self.__handler_send_to_single_target, } self.__broadcast_with_self_target_methods = { "AddTx", "AddTxList", "BroadcastVote" } self.stored_tx = queue.Queue() self.__timer_service = TimerService() @property def is_running(self): return self.__timer_service.is_run() def start(self): self.__timer_service.start() def stop(self): if self.__timer_service.is_run(): self.__timer_service.stop() self.__timer_service.wait() def handle_command(self, command, params): func = self.__handler_map[command] func(params) def __keep_grpc_connection(self, result, timeout, stub_manager: StubManager): return isinstance(result, _Rendezvous) \ and result.code() in (grpc.StatusCode.DEADLINE_EXCEEDED, grpc.StatusCode.UNAVAILABLE) \ and stub_manager.elapsed_last_succeed_time() < timeout def __broadcast_retry_async(self, peer_target, method_name, method_param, retry_times, timeout, stub, result): if isinstance(result, _Rendezvous) and result.code() == grpc.StatusCode.OK: return if isinstance(result, futures.Future) and not result.exception(): return logging.debug(f"try retry to : peer_target({peer_target})\n") if retry_times > 0: try: stub_manager: StubManager = self.__audience[peer_target] if stub_manager is None: logging.warning( f"broadcast_thread:__broadcast_retry_async Failed to connect to ({peer_target})." ) return retry_times -= 1 is_stub_reuse = stub_manager.stub != stub or self.__keep_grpc_connection( result, timeout, stub_manager) self.__call_async_to_target(peer_target, method_name, method_param, is_stub_reuse, retry_times, timeout) except KeyError as e: logging.debug( f"broadcast_thread:__broadcast_retry_async ({peer_target}) not in audience. ({e})" ) else: if isinstance(result, _Rendezvous): exception = result.details() elif isinstance(result, futures.Future): exception = result.exception() logging.warning(f"__broadcast_run_async fail({result})\n" f"cause by: {exception}\n" f"peer_target({peer_target})\n" f"method_name({method_name})\n" f"retry_remains({retry_times})\n" f"timeout({timeout})") def __call_async_to_target(self, peer_target, method_name, method_param, is_stub_reuse, retry_times, timeout): try: stub_manager: StubManager = self.__audience[peer_target] if stub_manager is None: logging.debug( f"broadcast_thread:__call_async_to_target Failed to connect to ({peer_target})." ) return call_back_partial = partial(self.__broadcast_retry_async, peer_target, method_name, method_param, retry_times, timeout, stub_manager.stub) stub_manager.call_async(method_name=method_name, message=method_param, is_stub_reuse=is_stub_reuse, call_back=call_back_partial, timeout=timeout) except KeyError as e: logging.debug( f"broadcast_thread:__call_async_to_target ({peer_target}) not in audience. ({e})" ) def __broadcast_run_async(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times # logging.debug(f"broadcast({method_name}) async... ({len(self.__audience)})") for target in self.__get_broadcast_targets(method_name): # util.logger.debug(f"method_name({method_name}), peer_target({target})") self.__call_async_to_target(target, method_name, method_param, True, retry_times, timeout) def __broadcast_run_sync(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ # logging.debug(f"broadcast({method_name}) sync... ({len(self.__audience)})") if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times for target in self.__get_broadcast_targets(method_name): try: stub_manager: StubManager = self.__audience[target] if stub_manager is None: logging.debug( f"broadcast_thread:__broadcast_run_sync Failed to connect to ({target})." ) continue response = stub_manager.call_in_times(method_name=method_name, message=method_param, timeout=timeout, retry_times=retry_times) if response is None: logging.warning( f"broadcast_thread:__broadcast_run_sync fail ({method_name}) " f"target({target}) ") except KeyError as e: logging.debug( f"broadcast_thread:__broadcast_run_sync ({target}) not in audience. ({e})" ) def __handler_send_to_single_target(self, param): method_name = param[0] method_param = param[1] target = param[2] self.__call_async_to_target(target, method_name, method_param, True, 0, conf.GRPC_TIMEOUT_BROADCAST_RETRY) def __add_audience(self, audience_target): util.logger.debug(f"audience_target({audience_target})") if audience_target not in self.__audience: stub_manager = StubManager(audience_target, loopchain_pb2_grpc.PeerServiceStub, ssl_auth_type=conf.GRPC_SSL_TYPE) self.__audience[audience_target] = stub_manager def __handler_update_audience(self, audience_targets): old_audience = self.__audience.copy() for audience_target in audience_targets: self.__add_audience(audience_target) old_audience.pop(audience_target, None) for old_audience_target in old_audience: old_stubmanager: StubManager = self.__audience.pop( old_audience_target, None) # TODO If necessary, close grpc with old_stubmanager. If not necessary just remove this comment. def __handler_broadcast(self, broadcast_param): # util.logger.debug(f"BroadcastThread received broadcast command") broadcast_method_name = broadcast_param[0] broadcast_method_param = broadcast_param[1] broadcast_method_kwparam = broadcast_param[2] # util.logger.debug("BroadcastThread method name: " + broadcast_method_name) # util.logger.debug("BroadcastThread method param: " + str(broadcast_method_param)) self.__broadcast_run(broadcast_method_name, broadcast_method_param, **broadcast_method_kwparam) def __make_tx_list_message(self): tx_list = [] tx_list_size = 0 tx_list_count = 0 remains = False while not self.stored_tx.empty(): stored_tx_item = self.stored_tx.get() tx_list_size += len(stored_tx_item) tx_list_count += 1 if tx_list_size >= conf.MAX_TX_SIZE_IN_BLOCK or tx_list_count >= conf.MAX_TX_COUNT_IN_ADDTX_LIST: self.stored_tx.put(stored_tx_item) remains = True break tx_list.append(stored_tx_item.get_tx_message()) message = loopchain_pb2.TxSendList(channel=self.__channel, tx_list=tx_list) return remains, message def __send_tx_by_timer(self, **kwargs): # util.logger.spam(f"broadcast_scheduler:__send_tx_by_timer") if self.__thread_variables[ self. THREAD_VARIABLE_PEER_STATUS] == PeerThreadStatus.leader_complained: logging.warning( "Leader is complained your tx just stored in queue by temporally: " + str(self.stored_tx.qsize())) else: # Send single tx for test # stored_tx_item = self.stored_tx.get() # self.__broadcast_run("AddTx", stored_tx_item.get_tx_message()) # Send multiple tx remains, message = self.__make_tx_list_message() self.__broadcast_run("AddTxList", message) if remains: self.__send_tx_in_timer() def __send_tx_in_timer(self, tx_item=None): # util.logger.spam(f"broadcast_scheduler:__send_tx_in_timer") duration = 0 if tx_item: self.stored_tx.put(tx_item) duration = conf.SEND_TX_LIST_DURATION if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list: self.__timer_service.add_timer( TimerService.TIMER_KEY_ADD_TX, Timer(target=TimerService.TIMER_KEY_ADD_TX, duration=duration, callback=self.__send_tx_by_timer, callback_kwargs={})) else: pass def __handler_create_tx(self, create_tx_param): # logging.debug(f"Broadcast create_tx....") try: tx_item = TxItem.create_tx_item(create_tx_param, self.__channel) except Exception as e: logging.warning(f"tx in channel({self.__channel})") logging.warning(f"__handler_create_tx: meta({create_tx_param})") logging.warning(f"tx dumps fail ({e})") return self.__send_tx_in_timer(tx_item) def __get_broadcast_targets(self, method_name): peer_targets = list(self.__audience) if self.__self_target is not None and method_name not in self.__broadcast_with_self_target_methods: peer_targets.remove(self.__self_target) return peer_targets
class _Broadcaster: """broadcast class for each channel""" THREAD_INFO_KEY = "thread_info" THREAD_VARIABLE_STUB_TO_SELF_PEER = "stub_to_self_peer" THREAD_VARIABLE_PEER_STATUS = "peer_status" SELF_PEER_TARGET_KEY = "self_peer_target" LEADER_PEER_TARGET_KEY = "leader_peer_target" def __init__(self, channel: str, self_target: str = None): self.__channel = channel self.__self_target = self_target self.__audience = {} # self.__audience[peer_target] = stub_manager self.__thread_variables = dict() self.__thread_variables[ self.THREAD_VARIABLE_PEER_STATUS] = PeerThreadStatus.normal if conf.IS_BROADCAST_ASYNC: self.__broadcast_run = self.__broadcast_run_async else: self.__broadcast_run = self.__broadcast_run_sync self.__handler_map = { BroadcastCommand.CREATE_TX: self.__handler_create_tx, BroadcastCommand.CONNECT_TO_LEADER: self.__handler_connect_to_leader, BroadcastCommand.SUBSCRIBE: self.__handler_subscribe, BroadcastCommand.UNSUBSCRIBE: self.__handler_unsubscribe, BroadcastCommand.BROADCAST: self.__handler_broadcast, BroadcastCommand.MAKE_SELF_PEER_CONNECTION: self.__handler_connect_to_self_peer, } self.__broadcast_with_self_target_methods = { "AddTx", "AddTxList", "BroadcastVote" } self.stored_tx = queue.Queue() self.__timer_service = TimerService() @property def is_running(self): return self.__timer_service.is_run() def start(self): self.__timer_service.start() def stop(self): if self.__timer_service.is_run(): self.__timer_service.stop() self.__timer_service.wait() def handle_command(self, command, params): func = self.__handler_map[command] func(params) def __keep_grpc_connection(self, result, timeout, stub_manager: StubManager): return isinstance(result, _Rendezvous) \ and result.code() in (grpc.StatusCode.DEADLINE_EXCEEDED, grpc.StatusCode.UNAVAILABLE) \ and stub_manager.elapsed_last_succeed_time() < timeout def __broadcast_retry_async(self, peer_target, method_name, method_param, retry_times, timeout, stub, result): if isinstance(result, _Rendezvous) and result.code() == grpc.StatusCode.OK: return if isinstance(result, futures.Future) and not result.exception(): return logging.debug(f"try retry to : peer_target({peer_target})\n") if retry_times > 0: try: stub_manager: StubManager = self.__audience[peer_target] if stub_manager is None: logging.warning( f"broadcast_thread:__broadcast_retry_async Failed to connect to ({peer_target})." ) return retry_times -= 1 is_stub_reuse = stub_manager.stub != stub or self.__keep_grpc_connection( result, timeout, stub_manager) self.__call_async_to_target(peer_target, method_name, method_param, is_stub_reuse, retry_times, timeout) except KeyError as e: logging.debug( f"broadcast_thread:__broadcast_retry_async ({peer_target}) not in audience. ({e})" ) else: if isinstance(result, _Rendezvous): exception = result.details() elif isinstance(result, futures.Future): exception = result.exception() logging.warning(f"__broadcast_run_async fail({result})\n" f"cause by: {exception}\n" f"peer_target({peer_target})\n" f"method_name({method_name})\n" f"retry_remains({retry_times})\n" f"timeout({timeout})") def __call_async_to_target(self, peer_target, method_name, method_param, is_stub_reuse, retry_times, timeout): try: stub_manager: StubManager = self.__audience[peer_target] if stub_manager is None: logging.debug( f"broadcast_thread:__call_async_to_target Failed to connect to ({peer_target})." ) return call_back_partial = partial(self.__broadcast_retry_async, peer_target, method_name, method_param, retry_times, timeout, stub_manager.stub) stub_manager.call_async(method_name=method_name, message=method_param, is_stub_reuse=is_stub_reuse, call_back=call_back_partial, timeout=timeout) except KeyError as e: logging.debug( f"broadcast_thread:__call_async_to_target ({peer_target}) not in audience. ({e})" ) def __broadcast_run_async(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times # logging.debug(f"broadcast({method_name}) async... ({len(self.__audience)})") for target in self.__get_broadcast_targets(method_name): # util.logger.debug(f"method_name({method_name}), peer_target({target})") self.__call_async_to_target(target, method_name, method_param, True, retry_times, timeout) def __broadcast_run_sync(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ # logging.debug(f"broadcast({method_name}) sync... ({len(self.__audience)})") if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times for target in self.__get_broadcast_targets(method_name): try: stub_manager: StubManager = self.__audience[target] if stub_manager is None: logging.debug( f"broadcast_thread:__broadcast_run_sync Failed to connect to ({target})." ) continue response = stub_manager.call_in_times(method_name=method_name, message=method_param, timeout=timeout, retry_times=retry_times) if response is None: logging.warning( f"broadcast_thread:__broadcast_run_sync fail ({method_name}) " f"target({target}) ") except KeyError as e: logging.debug( f"broadcast_thread:__broadcast_run_sync ({target}) not in audience. ({e})" ) def __handler_subscribe(self, audience_target): logging.debug( "BroadcastThread received subscribe command peer_target: " + str(audience_target)) if audience_target not in self.__audience: stub_manager = StubManager.get_stub_manager_to_server( audience_target, loopchain_pb2_grpc.PeerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT_WHEN_INITIAL, is_allow_null_stub=True, ssl_auth_type=conf.GRPC_SSL_TYPE) self.__audience[audience_target] = stub_manager def __handler_unsubscribe(self, audience_target): # logging.debug(f"BroadcastThread received unsubscribe command peer_target({unsubscribe_peer_target})") try: del self.__audience[audience_target] except KeyError: logging.warning(f"Already deleted peer: {audience_target}") def __handler_broadcast(self, broadcast_param): # logging.debug("BroadcastThread received broadcast command") broadcast_method_name = broadcast_param[0] broadcast_method_param = broadcast_param[1] broadcast_method_kwparam = broadcast_param[2] # logging.debug("BroadcastThread method name: " + broadcast_method_name) # logging.debug("BroadcastThread method param: " + str(broadcast_method_param)) self.__broadcast_run(broadcast_method_name, broadcast_method_param, **broadcast_method_kwparam) def __make_tx_list_message(self): tx_list = [] tx_list_size = 0 tx_list_count = 0 remains = False while not self.stored_tx.empty(): stored_tx_item = self.stored_tx.get() tx_list_size += len(stored_tx_item) tx_list_count += 1 if tx_list_size >= conf.MAX_TX_SIZE_IN_BLOCK or tx_list_count >= conf.MAX_TX_COUNT_IN_ADDTX_LIST: self.stored_tx.put(stored_tx_item) remains = True break tx_list.append(stored_tx_item.get_tx_message()) message = loopchain_pb2.TxSendList(channel=self.__channel, tx_list=tx_list) return remains, message def __send_tx_by_timer(self, **kwargs): # util.logger.spam(f"broadcast_scheduler:__send_tx_by_timer") if self.__thread_variables[ self. THREAD_VARIABLE_PEER_STATUS] == PeerThreadStatus.leader_complained: logging.warning( "Leader is complained your tx just stored in queue by temporally: " + str(self.stored_tx.qsize())) else: # Send single tx for test # stored_tx_item = self.stored_tx.get() # self.__broadcast_run("AddTx", stored_tx_item.get_tx_message()) # Send multiple tx remains, message = self.__make_tx_list_message() self.__broadcast_run("AddTxList", message) if remains: self.__send_tx_in_timer() def __send_tx_in_timer(self, tx_item=None): # util.logger.spam(f"broadcast_scheduler:__send_tx_in_timer") duration = 0 if tx_item: self.stored_tx.put(tx_item) duration = conf.SEND_TX_LIST_DURATION if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list: self.__timer_service.add_timer( TimerService.TIMER_KEY_ADD_TX, Timer(target=TimerService.TIMER_KEY_ADD_TX, duration=duration, callback=self.__send_tx_by_timer, callback_kwargs={})) else: pass def __handler_create_tx(self, create_tx_param): # logging.debug(f"Broadcast create_tx....") try: tx_item = TxItem.create_tx_item(create_tx_param, self.__channel) except Exception as e: logging.warning(f"tx in channel({self.__channel})") logging.warning(f"__handler_create_tx: meta({create_tx_param})") logging.warning(f"tx dumps fail ({e})") return self.__send_tx_in_timer(tx_item) def __handler_connect_to_leader(self, connect_to_leader_param): # logging.debug("(tx thread) try... connect to leader: " + str(connect_to_leader_param)) self.__thread_variables[ self.LEADER_PEER_TARGET_KEY] = connect_to_leader_param # stub_to_self_peer = __thread_variables[self.THREAD_VARIABLE_STUB_TO_SELF_PEER] self.__thread_variables[ self.THREAD_VARIABLE_PEER_STATUS] = PeerThreadStatus.normal def __handler_connect_to_self_peer(self, connect_param): # 자신을 생성한 부모 Peer 에 접속하기 위한 stub 을 만든다. # pipe 를 통한 return 은 pipe send 와 쌍이 맞지 않은 경우 오류를 발생시킬 수 있다. # 안전한 연결을 위하여 부모 프로세스와도 gRPC stub 을 이용하여 통신한다. logging.debug("try connect to self peer: " + str(connect_param)) stub_to_self_peer = StubManager.get_stub_manager_to_server( connect_param, loopchain_pb2_grpc.InnerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT_WHEN_INITIAL, is_allow_null_stub=True, ssl_auth_type=conf.SSLAuthType.none) self.__thread_variables[self.SELF_PEER_TARGET_KEY] = connect_param self.__thread_variables[ self.THREAD_VARIABLE_STUB_TO_SELF_PEER] = stub_to_self_peer def __get_broadcast_targets(self, method_name): peer_targets = list(self.__audience) if ObjectManager().rs_service: return peer_targets else: if self.__self_target is not None and method_name not in self.__broadcast_with_self_target_methods: peer_targets.remove(self.__self_target) return peer_targets
class ChannelService: def __init__(self, channel_name, amqp_target, amqp_key): self.__block_manager: BlockManager = None self.__score_container: CommonSubprocess = None self.__score_info: dict = None self.__peer_auth: PeerAuthorization = None self.__peer_manager: PeerManager = None self.__broadcast_scheduler: BroadcastScheduler = None self.__radio_station_stub = None self.__consensus: Consensus = None self.__proposer: Proposer = None self.__acceptor: Acceptor = None self.__timer_service = TimerService() loggers.get_preset().channel_name = channel_name loggers.get_preset().update_logger() channel_queue_name = conf.CHANNEL_QUEUE_NAME_FORMAT.format(channel_name=channel_name, amqp_key=amqp_key) self.__inner_service = ChannelInnerService( amqp_target, channel_queue_name, conf.AMQP_USERNAME, conf.AMQP_PASSWORD, channel_service=self) logging.info(f"ChannelService : {channel_name}, Queue : {channel_queue_name}") ChannelProperty().name = channel_name ChannelProperty().amqp_target = amqp_target StubCollection().amqp_key = amqp_key StubCollection().amqp_target = amqp_target command_arguments.add_raw_command(command_arguments.Type.Channel, channel_name) command_arguments.add_raw_command(command_arguments.Type.AMQPTarget, amqp_target) command_arguments.add_raw_command(command_arguments.Type.AMQPKey, amqp_key) ObjectManager().channel_service = self @property def block_manager(self): return self.__block_manager @property def score_container(self): return self.__score_container @property def score_info(self): return self.__score_info @property def radio_station_stub(self): return self.__radio_station_stub @property def peer_auth(self): return self.__peer_auth @property def peer_manager(self): return self.__peer_manager @property def broadcast_scheduler(self): return self.__broadcast_scheduler @property def consensus(self): return self.__consensus @property def acceptor(self): return self.__acceptor @property def timer_service(self): return self.__timer_service def serve(self): async def _serve(): await StubCollection().create_peer_stub() results = await StubCollection().peer_stub.async_task().get_channel_info_detail(ChannelProperty().name) await self.init(*results) self.__timer_service.start() logging.info(f'channel_service: init complete channel: {ChannelProperty().name}') loop = MessageQueueService.loop loop.create_task(_serve()) loop.add_signal_handler(signal.SIGINT, self.close) loop.add_signal_handler(signal.SIGTERM, self.close) try: loop.run_forever() finally: loop.run_until_complete(loop.shutdown_asyncgens()) loop.close() self.cleanup() def close(self): MessageQueueService.loop.stop() def cleanup(self): logging.info("Cleanup Channel Resources.") if self.__block_manager: self.__block_manager.stop() self.__block_manager.wait() self.__block_manager = None logging.info("Cleanup BlockManager.") if self.__score_container: self.__score_container.stop() self.__score_container.wait() self.__score_container = None logging.info("Cleanup ScoreContainer.") if self.__broadcast_scheduler: self.__broadcast_scheduler.stop() self.__broadcast_scheduler.wait() self.__broadcast_scheduler = None logging.info("Cleanup BroadcastSchuduler.") if self.__consensus: self.__consensus.stop() self.__consensus.wait() logging.info("Cleanup Consensus.") if self.__timer_service.is_run(): self.__timer_service.stop() self.__timer_service.wait() logging.info("Cleanup TimerSerivce.") async def init(self, peer_port, peer_target, rest_target, radio_station_target, peer_id, group_id, node_type, score_package): loggers.get_preset().peer_id = peer_id loggers.get_preset().update_logger() ChannelProperty().peer_port = peer_port ChannelProperty().peer_target = peer_target ChannelProperty().rest_target = rest_target ChannelProperty().radio_station_target = radio_station_target ChannelProperty().peer_id = peer_id ChannelProperty().group_id = group_id ChannelProperty().node_type = conf.NodeType(node_type) ChannelProperty().score_package = score_package self.__init_peer_auth() self.__init_block_manager() self.__init_broadcast_scheduler() self.__init_radio_station_stub() await self.__init_score_container() await self.__inner_service.connect(conf.AMQP_CONNECTION_ATTEMPS, conf.AMQP_RETRY_DELAY, exclusive=True) self.__peer_manager = PeerManager(ChannelProperty().name) if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: util.logger.spam(f"init consensus !") # load consensus self.__init_consensus() # load proposer self.__init_proposer(peer_id=peer_id) # load acceptor self.__init_acceptor(peer_id=peer_id) if self.is_support_node_function(conf.NodeFunction.Vote): self.connect_to_radio_station() await self.set_peer_type_in_channel() self.generate_genesis_block() if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: self.__consensus.change_epoch(precommit_block=self.__block_manager.get_blockchain().last_block) self.__consensus.start() elif conf.ALLOW_MAKE_EMPTY_BLOCK: self.block_manager.block_generation_scheduler.start() def __init_peer_auth(self): try: channel_authorization = IcxAuthorization if util.channel_use_icx(ChannelProperty().name) \ else PeerAuthorization self.__peer_auth = channel_authorization(ChannelProperty().name) except Exception as e: logging.exception(f"peer auth init fail cause : {e}") util.exit_and_msg(f"peer auth init fail cause : {e}") def __init_block_manager(self): logging.debug(f"__load_block_manager_each channel({ChannelProperty().name})") try: self.__block_manager = BlockManager( channel_manager=self, peer_id=ChannelProperty().peer_id, channel_name=ChannelProperty().name, level_db_identity=ChannelProperty().peer_target ) self.__block_manager.consensus_algorithm = self.__init_consensus_algorithm() if conf.CONSENSUS_ALGORITHM != conf.ConsensusAlgorithm.lft: self.__block_manager.start() except leveldb.LevelDBError as e: util.exit_and_msg("LevelDBError(" + str(e) + ")") def __init_consensus(self): consensus = Consensus(self, ChannelProperty().name) self.__consensus = consensus self.__block_manager.consensus = consensus consensus.multiple_register(self.__block_manager) def __init_proposer(self, peer_id: str): proposer = Proposer( name="loopchain.consensus.Proposer", peer_id=peer_id, channel=ChannelProperty().name, channel_service=self) self.__consensus.multiple_register(proposer) self.__proposer = proposer def __init_acceptor(self, peer_id: str): acceptor = Acceptor( name="loopchain.consensus.Acceptor", consensus=self.__consensus, peer_id=peer_id, channel=ChannelProperty().name, channel_service=self) self.__consensus.multiple_register(acceptor) self.__acceptor = acceptor def __init_broadcast_scheduler(self): scheduler = BroadcastScheduler(channel=ChannelProperty().name, self_target=ChannelProperty().peer_target) scheduler.start() self.__broadcast_scheduler = scheduler future = scheduler.schedule_job(BroadcastCommand.SUBSCRIBE, ChannelProperty().peer_target) future.result(conf.TIMEOUT_FOR_FUTURE) def __init_radio_station_stub(self): if self.is_support_node_function(conf.NodeFunction.Vote): self.__radio_station_stub = StubManager.get_stub_manager_to_server( ChannelProperty().radio_station_target, loopchain_pb2_grpc.RadioStationStub, conf.CONNECTION_RETRY_TIMEOUT_TO_RS, ssl_auth_type=conf.GRPC_SSL_TYPE) else: self.__radio_station_stub = RestStubManager(ChannelProperty().radio_station_target, ChannelProperty().name) async def __init_score_container(self): """create score container and save score_info and score_stub """ for i in range(conf.SCORE_LOAD_RETRY_TIMES): try: self.__score_info = await self.__run_score_container() except BaseException as e: util.logger.spam(f"channel_manager:load_score_container_each score_info load fail retry({i})") logging.error(e) traceback.print_exc() time.sleep(conf.SCORE_LOAD_RETRY_INTERVAL) # This blocking main thread is intended. else: break async def __run_score_container(self): if not conf.USE_EXTERNAL_SCORE or conf.EXTERNAL_SCORE_RUN_IN_LAUNCHER: process_args = ['python3', '-m', 'loopchain', 'score', '--channel', ChannelProperty().name, '--score_package', ChannelProperty().score_package] process_args += command_arguments.get_raw_commands_by_filter( command_arguments.Type.AMQPTarget, command_arguments.Type.AMQPKey, command_arguments.Type.Develop, command_arguments.Type.ConfigurationFilePath ) self.__score_container = CommonSubprocess(process_args) if util.channel_use_icx(ChannelProperty().name): await StubCollection().create_icon_score_stub(ChannelProperty().name) await StubCollection().icon_score_stubs[ChannelProperty().name].connect() await StubCollection().icon_score_stubs[ChannelProperty().name].async_task().hello() return None else: await StubCollection().create_score_stub(ChannelProperty().name, ChannelProperty().score_package) await StubCollection().score_stubs[ChannelProperty().name].connect() await StubCollection().score_stubs[ChannelProperty().name].async_task().hello() return await self.__load_score() def __init_consensus_algorithm(self): """initialize a consensus algorithm by configuration. """ if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.none: consensus_algorithm = ConsensusNone(self.__block_manager) elif conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.siever: consensus_algorithm = ConsensusSiever(self.__block_manager) elif conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: consensus_algorithm = None else: consensus_algorithm = ConsensusDefault(self.__block_manager) return consensus_algorithm async def __load_score(self): channel_name = ChannelProperty().name score_package_name = ChannelProperty().score_package util.logger.spam(f"peer_service:__load_score --init--") logging.info("LOAD SCORE AND CONNECT TO SCORE SERVICE!") params = dict() params[message_code.MetaParams.ScoreLoad.repository_path] = conf.DEFAULT_SCORE_REPOSITORY_PATH params[message_code.MetaParams.ScoreLoad.score_package] = score_package_name params[message_code.MetaParams.ScoreLoad.base] = conf.DEFAULT_SCORE_BASE params[message_code.MetaParams.ScoreLoad.peer_id] = ChannelProperty().peer_id meta = json.dumps(params) logging.debug(f"load score params : {meta}") util.logger.spam(f"peer_service:__load_score --1--") score_stub = StubCollection().score_stubs[channel_name] response = await score_stub.async_task().score_load(meta) logging.debug("try score load on score service: " + str(response)) if not response: return None if response.code != message_code.Response.success: util.exit_and_msg("Fail Get Score from Score Server...") return None logging.debug("Get Score from Score Server...") score_info = json.loads(response.meta) logging.info("LOAD SCORE DONE!") util.logger.spam(f"peer_service:__load_score --end--") return score_info def is_support_node_function(self, node_function): return conf.NodeType.is_support_node_function(node_function, ChannelProperty().node_type) def get_channel_option(self) -> dict: channel_option = conf.CHANNEL_OPTION return channel_option[ChannelProperty().name] def generate_genesis_block(self): if self.block_manager.peer_type != loopchain_pb2.BLOCK_GENERATOR: return block_chain = self.block_manager.get_blockchain() if block_chain.block_height > -1: logging.debug("genesis block was already generated") return block_chain.generate_genesis_block() def connect_to_radio_station(self, is_reconnect=False): response = self.__radio_station_stub.call_in_times( method_name="ConnectPeer", message=loopchain_pb2.ConnectPeerRequest( channel=ChannelProperty().name, peer_object=b'', peer_id=ChannelProperty().peer_id, peer_target=ChannelProperty().peer_target, group_id=ChannelProperty().group_id, cert=self.peer_auth.peer_cert), retry_times=conf.CONNECTION_RETRY_TIMES_TO_RS, is_stub_reuse=True, timeout=conf.CONNECTION_TIMEOUT_TO_RS) # start next ConnectPeer timer if TimerService.TIMER_KEY_CONNECT_PEER not in self.__timer_service.timer_list.keys(): self.__timer_service.add_timer( TimerService.TIMER_KEY_CONNECT_PEER, Timer( target=TimerService.TIMER_KEY_CONNECT_PEER, duration=conf.CONNECTION_RETRY_TIMER, callback=self.connect_to_radio_station, callback_kwargs={"is_reconnect": True} ) ) if is_reconnect: return if response and response.status == message_code.Response.success: peer_list_data = pickle.loads(response.peer_list) self.__peer_manager.load(peer_list_data, False) peers, peer_list = self.__peer_manager.get_peers_for_debug() logging.debug("peer list update: " + peers) # add connected peer to processes audience for each_peer in peer_list: util.logger.spam(f"peer_service:connect_to_radio_station peer({each_peer.target}-{each_peer.status})") if each_peer.status == PeerStatus.connected: self.__broadcast_scheduler.schedule_job(BroadcastCommand.SUBSCRIBE, each_peer.target) async def subscribe_to_radio_station(self): await self.__subscribe_call_to_stub_by_method(self.__radio_station_stub, loopchain_pb2.PEER) async def subscribe_to_target_stub(self, target_stub): await self.__subscribe_call_to_stub_by_method(target_stub, loopchain_pb2.PEER) async def subscribe_to_peer(self, peer_id, peer_type): peer = self.peer_manager.get_peer(peer_id) peer_stub = self.peer_manager.get_peer_stub_manager(peer) await self.__subscribe_call_to_stub_by_method(peer_stub, peer_type) self.__broadcast_scheduler.schedule_job(BroadcastCommand.SUBSCRIBE, peer_stub.target) async def __subscribe_call_to_stub_by_method(self, peer_stub, peer_type): if self.is_support_node_function(conf.NodeFunction.Vote): await peer_stub.call_async( "Subscribe", loopchain_pb2.PeerRequest( channel=ChannelProperty().name, peer_target=ChannelProperty().peer_target, peer_type=peer_type, peer_id=ChannelProperty().peer_id, group_id=ChannelProperty().group_id, node_type=ChannelProperty().node_type ), ) else: util.logger.spam(f"channel_service:__subscribe_call_to_stub_by_method " f"peer_target({ChannelProperty().rest_target})") response = self.__subscribe_call_to_rs_stub(peer_stub) if response['response_code'] != message_code.Response.success: error = f"subscribe fail to peer_target({ChannelProperty().radio_station_target}) " \ f"reason({response['message']})" await StubCollection().peer_stub.async_task().stop(message=error) def __subscribe_call_to_rs_stub(self, rs_rest_stub): response = {'response_code': message_code.Response.fail, 'message': message_code.get_response_msg(message_code.Response.fail)} try: if conf.REST_SSL_TYPE == conf.SSLAuthType.none: peer_target = ChannelProperty().rest_target else: peer_target = f"https://{ChannelProperty().rest_target}" response = rs_rest_stub.call( "Subscribe", { 'channel': ChannelProperty().name, 'peer_target': peer_target } ) except Exception as e: logging.warning(f"Due to Subscription fail to RadioStation(mother peer), " f"automatically retrying subscribe call") if response['response_code'] == message_code.Response.success: if TimerService.TIMER_KEY_SUBSCRIBE in self.__timer_service.timer_list.keys(): self.__timer_service.stop_timer(TimerService.TIMER_KEY_SUBSCRIBE) self.radio_station_stub.update_methods_version() logging.debug(f"Subscription to RadioStation(mother peer) is successful.") if TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE in self.__timer_service.timer_list.keys(): self.__timer_service.stop_timer(TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE) # start next get_status timer timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key not in self.__timer_service.timer_list.keys(): util.logger.spam(f"add timer for check_block_height_call to radiostation...") self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.GET_LAST_BLOCK_TIMER, is_repeat=True, callback=self.__check_block_height_call_to_rs_stub, callback_kwargs={"rs_rest_stub": rs_rest_stub} ) ) else: timer_key = TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE if timer_key not in self.__timer_service.timer_list.keys(): error = f"Shutdown by Subscribe retry timeout({conf.SHUTDOWN_TIMER})" self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.SHUTDOWN_TIMER, callback=self.__shutdown_peer, callback_kwargs={"message": error} ) ) return response def __check_block_height_call_to_rs_stub(self, **kwargs): rs_rest_stub = kwargs.get("rs_rest_stub", None) response = dict() try: response = rs_rest_stub.call("GetLastBlock") except Exception as e: response['response_code'] = message_code.Response.fail if response['response_code'] == message_code.Response.success: if response['block']['height'] <= self.__block_manager.get_blockchain().block_height: # keep get last block timer, citizen subscription is still valid. return # citizen needs additional block or failed to connect to mother peer. timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key in self.__timer_service.timer_list.keys(): util.logger.spam(f"stop timer for check_block_height_call to radiostation...") self.__timer_service.stop_timer(timer_key) timer_key = TimerService.TIMER_KEY_SUBSCRIBE if timer_key not in self.__timer_service.timer_list.keys(): self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.SUBSCRIBE_RETRY_TIMER, is_repeat=True, callback=self.__subscribe_call_to_rs_stub, callback_kwargs={"rs_rest_stub": rs_rest_stub} ) ) def __shutdown_peer(self, **kwargs): util.logger.spam(f"channel_service:__shutdown_peer") StubCollection().peer_stub.sync_task().stop(message=kwargs['message']) def set_peer_type(self, peer_type): """Set peer type when peer init only :param peer_type: :return: """ self.__block_manager.set_peer_type(peer_type) def save_peer_manager(self, peer_manager): """peer_list 를 leveldb 에 저장한다. :param peer_manager: """ level_db_key_name = str.encode(conf.LEVEL_DB_KEY_FOR_PEER_LIST) try: dump = peer_manager.dump() level_db = self.__block_manager.get_level_db() level_db.Put(level_db_key_name, dump) except AttributeError as e: logging.warning("Fail Save Peer_list: " + str(e)) async def set_peer_type_in_channel(self): peer_type = loopchain_pb2.PEER peer_leader = self.peer_manager.get_leader_peer( is_complain_to_rs=self.is_support_node_function(conf.NodeFunction.Vote)) logging.debug(f"channel({ChannelProperty().name}) peer_leader: " + str(peer_leader)) logger_preset = loggers.get_preset() if self.is_support_node_function(conf.NodeFunction.Vote) and ChannelProperty().peer_id == peer_leader.peer_id: logger_preset.is_leader = True logging.debug(f"Set Peer Type Leader! channel({ChannelProperty().name})") peer_type = loopchain_pb2.BLOCK_GENERATOR else: logger_preset.is_leader = False logger_preset.update_logger() if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: self.consensus.leader_id = peer_leader.peer_id if peer_type == loopchain_pb2.BLOCK_GENERATOR: self.block_manager.set_peer_type(peer_type) self.__ready_to_height_sync(True) await self.subscribe_to_radio_station() elif peer_type == loopchain_pb2.PEER: self.__ready_to_height_sync(False) await self.__block_height_sync_channel() def __ready_to_height_sync(self, is_leader: bool = False): block_chain = self.block_manager.get_blockchain() block_chain.init_block_chain(is_leader) if block_chain.block_height > -1: self.block_manager.rebuild_block() async def __block_height_sync_channel(self): # leader 로 시작하지 않았는데 자신의 정보가 leader Peer 정보이면 block height sync 하여 # 최종 블럭의 leader 를 찾는다. peer_manager = self.peer_manager peer_leader = peer_manager.get_leader_peer() self_peer_object = peer_manager.get_peer(ChannelProperty().peer_id) is_delay_announce_new_leader = False peer_old_leader = None if peer_leader: block_sync_target = peer_leader.target block_sync_target_stub = StubManager.get_stub_manager_to_server( block_sync_target, loopchain_pb2_grpc.PeerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT, ssl_auth_type=conf.GRPC_SSL_TYPE ) else: block_sync_target = ChannelProperty().radio_station_target block_sync_target_stub = self.__radio_station_stub if block_sync_target != ChannelProperty().peer_target: if block_sync_target_stub is None: logging.warning("You maybe Older from this network... or No leader in this network!") is_delay_announce_new_leader = True peer_old_leader = peer_leader peer_leader = self.peer_manager.leader_complain_to_rs( conf.ALL_GROUP_ID, is_announce_new_peer=False) if peer_leader is not None and ChannelProperty().node_type == conf.NodeType.CommunityNode: block_sync_target_stub = StubManager.get_stub_manager_to_server( peer_leader.target, loopchain_pb2_grpc.PeerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT, ssl_auth_type=conf.GRPC_SSL_TYPE ) if self.is_support_node_function(conf.NodeFunction.Vote) and \ (not peer_leader or peer_leader.peer_id == ChannelProperty().peer_id): peer_leader = self_peer_object self.block_manager.set_peer_type(loopchain_pb2.BLOCK_GENERATOR) else: _, future = self.block_manager.block_height_sync(block_sync_target_stub) await future self.show_peers() if is_delay_announce_new_leader and ChannelProperty().node_type == conf.NodeType.CommunityNode: self.peer_manager.announce_new_leader( peer_old_leader.peer_id, peer_leader.peer_id, self_peer_id=ChannelProperty().peer_id) def show_peers(self): logging.debug(f"peer_service:show_peers ({ChannelProperty().name}): ") for peer in self.peer_manager.get_IP_of_peers_in_group(): logging.debug("peer_target: " + peer) async def reset_leader(self, new_leader_id, block_height=0): logging.info(f"RESET LEADER channel({ChannelProperty().name}) leader_id({new_leader_id})") complained_leader = self.peer_manager.get_leader_peer() leader_peer = self.peer_manager.get_peer(new_leader_id, None) if block_height > 0 and block_height != self.block_manager.get_blockchain().last_block.height + 1: logging.warning(f"height behind peer can not take leader role.") return if leader_peer is None: logging.warning(f"in peer_service:reset_leader There is no peer by peer_id({new_leader_id})") return util.logger.spam(f"peer_service:reset_leader target({leader_peer.target})") self_peer_object = self.peer_manager.get_peer(ChannelProperty().peer_id) self.peer_manager.set_leader_peer(leader_peer, None) peer_leader = self.peer_manager.get_leader_peer() peer_type = loopchain_pb2.PEER if self_peer_object.target == peer_leader.target: loggers.get_preset().is_leader = True loggers.get_preset().update_logger() logging.debug("Set Peer Type Leader!") peer_type = loopchain_pb2.BLOCK_GENERATOR self.block_manager.get_blockchain().reset_made_block_count() self.peer_manager.announce_new_leader( complained_leader.peer_id, new_leader_id, is_broadcast=True, self_peer_id=ChannelProperty().peer_id) else: loggers.get_preset().is_leader = False loggers.get_preset().update_logger() logging.debug("Set Peer Type Peer!") # 새 leader 에게 subscribe 하기 await self.subscribe_to_radio_station() await self.subscribe_to_peer(peer_leader.peer_id, loopchain_pb2.BLOCK_GENERATOR) # update candidate blocks self.block_manager.get_candidate_blocks().set_last_block(self.block_manager.get_blockchain().last_block) self.block_manager.set_peer_type(peer_type) def set_new_leader(self, new_leader_id, block_height=0): logging.info(f"SET NEW LEADER channel({ChannelProperty().name}) leader_id({new_leader_id})") # complained_leader = self.peer_manager.get_leader_peer() leader_peer = self.peer_manager.get_peer(new_leader_id, None) if block_height > 0 and block_height != self.block_manager.get_blockchain().last_block.height + 1: logging.warning(f"height behind peer can not take leader role.") return if leader_peer is None: logging.warning(f"in channel_service:set_new_leader::There is no peer by peer_id({new_leader_id})") return util.logger.spam(f"channel_service:set_new_leader::leader_target({leader_peer.target})") self_peer_object = self.peer_manager.get_peer(ChannelProperty().peer_id) self.peer_manager.set_leader_peer(leader_peer, None) peer_leader = self.peer_manager.get_leader_peer() if self_peer_object.target == peer_leader.target: loggers.get_preset().is_leader = True loggers.get_preset().update_logger() logging.debug("I'm Leader Peer!") else: loggers.get_preset().is_leader = False loggers.get_preset().update_logger() logging.debug("I'm general Peer!") # 새 leader 에게 subscribe 하기 # await self.subscribe_to_radio_station() # await self.subscribe_to_peer(peer_leader.peer_id, loopchain_pb2.BLOCK_GENERATOR) def genesis_invoke(self, block: Block) -> dict or None: if util.channel_use_icx(ChannelProperty().name): method = "icx_sendTransaction" transactions = [] for tx in block.confirmed_transaction_list: transaction = { "method": method, "params": { "txHash": tx.tx_hash }, "genesisData": tx.genesis_origin_data } transactions.append(transaction) request = { 'block': { 'blockHeight': block.height, 'blockHash': block.block_hash, 'timestamp': block.time_stamp }, 'transactions': transactions } request = convert_params(request, ParamType.invoke) stub = StubCollection().icon_score_stubs[ChannelProperty().name] response = stub.sync_task().invoke(request) response_to_json_query(response) block.commit_state[ChannelProperty().name] = response['stateRootHash'] return response["txResults"] else: block_object = pickle.dumps(block) stub = StubCollection().score_stubs[ChannelProperty().name] response = stub.sync_task().genesis_invoke(block_object) if response.code == message_code.Response.success: return json.loads(response.meta) return None def score_invoke(self, _block: Block) -> dict or None: if util.channel_use_icx(ChannelProperty().name): method = "icx_sendTransaction" transactions = [] for tx in _block.confirmed_transaction_list: data = tx.icx_origin_data transaction = { "method": method, "params": data } transactions.append(transaction) request = { 'block': { 'blockHeight': _block.height, 'blockHash': _block.block_hash, 'prevBlockHash': _block.prev_block_hash, 'timestamp': _block.time_stamp }, 'transactions': transactions } request = convert_params(request, ParamType.invoke) stub = StubCollection().icon_score_stubs[ChannelProperty().name] response = stub.sync_task().invoke(request) response_to_json_query(response) _block.commit_state[ChannelProperty().name] = response['stateRootHash'] return response["txResults"] else: stub = StubCollection().score_stubs[ChannelProperty().name] response = stub.sync_task().score_invoke(_block) if response.code == message_code.Response.success: commit_state = pickle.loads(response.object) _block.commit_state = commit_state return json.loads(response.meta) return None def score_change_block_hash(self, block_height, old_block_hash, new_block_hash): change_hash_info = json.dumps({"block_height": block_height, "old_block_hash": old_block_hash, "new_block_hash": new_block_hash}) if not util.channel_use_icx(ChannelProperty().name): stub = StubCollection().score_stubs[ChannelProperty().name] stub.sync_task().change_block_hash(change_hash_info) def score_write_precommit_state(self, block: Block): logging.debug(f"call score commit {ChannelProperty().name} {block.height} {block.block_hash}") if util.channel_use_icx(ChannelProperty().name): request = { "blockHeight": block.height, "blockHash": block.block_hash, } request = convert_params(request, ParamType.write_precommit_state) stub = StubCollection().icon_score_stubs[ChannelProperty().name] stub.sync_task().write_precommit_state(request) return True else: block_commit_info = json.dumps({"block_height": block.height, "block_hash": block.block_hash}) stub = StubCollection().score_stubs[ChannelProperty().name] response = stub.sync_task().write_precommit_state(block_commit_info) if response.code == message_code.Response.success: return True else: logging.error(f"score db commit fail cause {response.message}") return False def score_remove_precommit_state(self, block: Block): if not util.channel_use_icx(ChannelProperty().name): request = { "blockHeight": block.height, "blockHash": block.block_hash, } request = convert_params(request, ParamType.remove_precommit_state) stub = StubCollection().icon_score_stubs[ChannelProperty().name] stub.sync_task().remove_precommit_state(request) return True else: invoke_fail_info = json.dumps({"block_height": block.height, "block_hash": block.block_hash}) stub = StubCollection().score_stubs[ChannelProperty().name] stub.sync_task().remove_precommit_state(invoke_fail_info) return True def get_object_has_queue_by_consensus(self): if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: object_has_queue = self.__consensus else: object_has_queue = self.__block_manager return object_has_queue
class BroadcastScheduler(CommonThread): """broadcast class for each channel""" THREAD_INFO_KEY = "thread_info" THREAD_VARIABLE_STUB_TO_SELF_PEER = "stub_to_self_peer" THREAD_VARIABLE_PEER_STATUS = "peer_status" SELF_PEER_TARGET_KEY = "self_peer_target" LEADER_PEER_TARGET_KEY = "leader_peer_target" def __init__(self, channel="", self_target=""): super().__init__() self.__channel = channel self.__self_target = self_target self.__audience = {} # self.__audience[peer_target] = stub_manager self.__thread_variables = dict() self.__thread_variables[ self.THREAD_VARIABLE_PEER_STATUS] = PeerThreadStatus.normal if conf.IS_BROADCAST_ASYNC: self.__broadcast_run = self.__broadcast_run_async else: self.__broadcast_run = self.__broadcast_run_sync self.__handler_map = { BroadcastCommand.CREATE_TX: self.__handler_create_tx, BroadcastCommand.CONNECT_TO_LEADER: self.__handler_connect_to_leader, BroadcastCommand.SUBSCRIBE: self.__handler_subscribe, BroadcastCommand.UNSUBSCRIBE: self.__handler_unsubscribe, BroadcastCommand.UPDATE_AUDIENCE: self.__handler_update_audience, BroadcastCommand.BROADCAST: self.__handler_broadcast, BroadcastCommand.MAKE_SELF_PEER_CONNECTION: self.__handler_connect_to_self_peer, } self.__broadcast_with_self_target_methods = { "AddTx", "AddTxList", "BroadcastVote" } self.stored_tx = queue.Queue() self.__broadcast_pool = futures.ThreadPoolExecutor( conf.MAX_BROADCAST_WORKERS, "BroadcastThread") self.__broadcast_queue = queue.PriorityQueue() self.__timer_service = TimerService() def stop(self): super().stop() self.__broadcast_queue.put((None, None, None, None)) self.__broadcast_pool.shutdown(False) if self.__timer_service.is_run(): self.__timer_service.stop() self.__timer_service.wait() def run(self, event: threading.Event): event.set() self.__timer_service.start() def _callback(curr_future: futures.Future, executor_future: futures.Future): if executor_future.exception(): curr_future.set_exception(executor_future.exception()) logging.error(executor_future.exception()) else: curr_future.set_result(executor_future.result()) while self.is_run(): priority, command, params, future = self.__broadcast_queue.get() if command is None: break func = self.__handler_map[command] return_future = self.__broadcast_pool.submit(func, params) return_future.add_done_callback(partial(_callback, future)) def schedule_job(self, command, params): if command == BroadcastCommand.CREATE_TX: priority = (10, time.time()) elif isinstance(params, tuple) and params[0] == "AddTx": priority = (10, time.time()) else: priority = (0, time.time()) future = futures.Future() self.__broadcast_queue.put((priority, command, params, future)) util.logger.spam( f"broadcast_scheduler:schedule_job qsize({self.__broadcast_queue.qsize()})" ) return future def schedule_broadcast(self, method_name, method_param, *, retry_times=None, timeout=None): """등록된 모든 Peer 의 동일한 gRPC method 를 같은 파라미터로 호출한다. """ # logging.warning("broadcast in process ==========================") # logging.debug("pickle method_param: " + str(pickle.dumps(method_param))) kwargs = {} if retry_times is not None: kwargs['retry_times'] = retry_times if timeout is not None: kwargs['timeout'] = timeout self.schedule_job(BroadcastCommand.BROADCAST, (method_name, method_param, kwargs)) def __broadcast_retry_async(self, peer_target, method_name, method_param, retry_times, timeout, result): if isinstance(result, _Rendezvous) and result.code() == grpc.StatusCode.OK: return if isinstance(result, futures.Future) and not result.exception(): return logging.debug(f"try retry to : peer_target({peer_target})\n") if retry_times > 0: retry_times -= 1 self.__call_async_to_target(peer_target, method_name, method_param, False, retry_times, timeout) else: if isinstance(result, _Rendezvous): exception = result.details() elif isinstance(result, futures.Future): exception = result.exception() logging.warning(f"__broadcast_run_async fail({result})\n" f"cause by: {exception}\n" f"peer_target({peer_target})\n" f"method_name({method_name})\n" f"retry_remains({retry_times})\n" f"timeout({timeout})") def __call_async_to_target(self, peer_target, method_name, method_param, is_stub_reuse, retry_times, timeout): try: call_back_partial = None stub_item = None if peer_target in self.__audience.keys(): call_back_partial = partial(self.__broadcast_retry_async, peer_target, method_name, method_param, retry_times, timeout) stub_item = self.__audience[peer_target] except KeyError as e: logging.debug( f"broadcast_thread:__call_async_to_target ({peer_target}) not in audience. ({e})" ) else: if stub_item: stub_item.call_async(method_name=method_name, message=method_param, is_stub_reuse=is_stub_reuse, call_back=call_back_partial, timeout=timeout) def __broadcast_run_async(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times # logging.debug(f"broadcast({method_name}) async... ({len(self.__audience)})") for target in self.__get_broadcast_targets(method_name): # util.logger.debug(f"method_name({method_name}), peer_target({target})") self.__call_async_to_target(target, method_name, method_param, True, retry_times, timeout) def __broadcast_run_sync(self, method_name, method_param, retry_times=None, timeout=None): """call gRPC interface of audience :param method_name: gRPC interface :param method_param: gRPC message """ # logging.debug(f"broadcast({method_name}) sync... ({len(self.__audience)})") if timeout is None: timeout = conf.GRPC_TIMEOUT_BROADCAST_RETRY retry_times = conf.BROADCAST_RETRY_TIMES if retry_times is None else retry_times for target in self.__get_broadcast_targets(method_name): if target in self.__audience.keys(): stub_item = self.__audience[target] response = stub_item.call_in_times(method_name=method_name, message=method_param, timeout=timeout, retry_times=retry_times) if response is None: logging.warning( f"broadcast_thread:__broadcast_run_sync fail ({method_name}) " f"target({target}) ") def __handler_subscribe(self, audience_target): logging.debug( "BroadcastThread received subscribe command peer_target: " + str(audience_target)) if audience_target not in self.__audience: stub_manager = StubManager.get_stub_manager_to_server( audience_target, loopchain_pb2_grpc.PeerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT_WHEN_INITIAL, is_allow_null_stub=True, ssl_auth_type=conf.GRPC_SSL_TYPE) self.__audience[audience_target] = stub_manager def __handler_unsubscribe(self, audience_target): # logging.debug(f"BroadcastThread received unsubscribe command peer_target({unsubscribe_peer_target})") try: del self.__audience[audience_target] except KeyError: logging.warning(f"Already deleted peer: {audience_target}") def __handler_update_audience(self, audience_param): util.logger.spam( f"broadcast_thread:__handler_update_audience audience_param({audience_param})" ) peer_manager = PeerManager(ChannelProperty().name) peer_list_data = pickle.loads(audience_param) peer_manager.load(peer_list_data, False) for peer_id in list(peer_manager.peer_list[conf.ALL_GROUP_ID]): peer_each = peer_manager.peer_list[conf.ALL_GROUP_ID][peer_id] if peer_each.target != self.__self_target: logging.warning( f"broadcast thread peer_targets({peer_each.target})") self.__handler_subscribe(peer_each.target) def __handler_broadcast(self, broadcast_param): # logging.debug("BroadcastThread received broadcast command") broadcast_method_name = broadcast_param[0] broadcast_method_param = broadcast_param[1] broadcast_method_kwparam = broadcast_param[2] # logging.debug("BroadcastThread method name: " + broadcast_method_name) # logging.debug("BroadcastThread method param: " + str(broadcast_method_param)) self.__broadcast_run(broadcast_method_name, broadcast_method_param, **broadcast_method_kwparam) def __make_tx_list_message(self): tx_list = [] tx_list_size = 0 tx_list_count = 0 remains = False while not self.stored_tx.empty(): stored_tx_item = self.stored_tx.get() tx_list_size += len(stored_tx_item) tx_list_count += 1 if tx_list_size >= conf.MAX_TX_SIZE_IN_BLOCK or tx_list_count >= conf.MAX_TX_COUNT_IN_ADDTX_LIST: self.stored_tx.put(stored_tx_item) remains = True break tx_list.append(stored_tx_item.get_tx_message()) message = loopchain_pb2.TxSendList(channel=self.__channel, tx_list=tx_list) return remains, message def __send_tx_by_timer(self, **kwargs): # util.logger.spam(f"broadcast_scheduler:__send_tx_by_timer") if self.__thread_variables[ self. THREAD_VARIABLE_PEER_STATUS] == PeerThreadStatus.leader_complained: logging.warning( "Leader is complained your tx just stored in queue by temporally: " + str(self.stored_tx.qsize())) else: # Send single tx for test # stored_tx_item = self.stored_tx.get() # self.__broadcast_run("AddTx", stored_tx_item.get_tx_message()) # Send multiple tx remains, message = self.__make_tx_list_message() self.__broadcast_run("AddTxList", message) if remains: self.__send_tx_in_timer() def __send_tx_in_timer(self, tx_item=None): # util.logger.spam(f"broadcast_scheduler:__send_tx_in_timer") duration = 0 if tx_item: self.stored_tx.put(tx_item) duration = conf.SEND_TX_LIST_DURATION if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list: self.__timer_service.add_timer( TimerService.TIMER_KEY_ADD_TX, Timer(target=TimerService.TIMER_KEY_ADD_TX, duration=duration, callback=self.__send_tx_by_timer, callback_kwargs={})) else: pass def __handler_create_tx(self, create_tx_param): # logging.debug(f"Broadcast create_tx....") try: tx_item = TxItem.create_tx_item(create_tx_param, self.__channel) except Exception as e: logging.warning(f"tx in channel({self.__channel})") logging.warning(f"__handler_create_tx: meta({create_tx_param})") logging.warning(f"tx dumps fail ({e})") return self.__send_tx_in_timer(tx_item) def __handler_connect_to_leader(self, connect_to_leader_param): # logging.debug("(tx thread) try... connect to leader: " + str(connect_to_leader_param)) self.__thread_variables[ self.LEADER_PEER_TARGET_KEY] = connect_to_leader_param # stub_to_self_peer = __thread_variables[self.THREAD_VARIABLE_STUB_TO_SELF_PEER] self.__thread_variables[ self.THREAD_VARIABLE_PEER_STATUS] = PeerThreadStatus.normal def __handler_connect_to_self_peer(self, connect_param): # 자신을 생성한 부모 Peer 에 접속하기 위한 stub 을 만든다. # pipe 를 통한 return 은 pipe send 와 쌍이 맞지 않은 경우 오류를 발생시킬 수 있다. # 안전한 연결을 위하여 부모 프로세스와도 gRPC stub 을 이용하여 통신한다. logging.debug("try connect to self peer: " + str(connect_param)) stub_to_self_peer = StubManager.get_stub_manager_to_server( connect_param, loopchain_pb2_grpc.InnerServiceStub, time_out_seconds=conf.CONNECTION_RETRY_TIMEOUT_WHEN_INITIAL, is_allow_null_stub=True, ssl_auth_type=conf.SSLAuthType.none) self.__thread_variables[self.SELF_PEER_TARGET_KEY] = connect_param self.__thread_variables[ self.THREAD_VARIABLE_STUB_TO_SELF_PEER] = stub_to_self_peer def __get_broadcast_targets(self, method_name): peer_targets = list(self.__audience) if ObjectManager().rs_service: return peer_targets else: if method_name not in self.__broadcast_with_self_target_methods: peer_targets.remove(ChannelProperty().peer_target) return peer_targets