def __subscribe_call_to_rs_stub(self, rs_rest_stub): response = {'response_code': message_code.Response.fail, 'message': message_code.get_response_msg(message_code.Response.fail)} try: if conf.REST_SSL_TYPE == conf.SSLAuthType.none: peer_target = ChannelProperty().rest_target else: peer_target = f"https://{ChannelProperty().rest_target}" response = rs_rest_stub.call( "Subscribe", { 'channel': ChannelProperty().name, 'peer_target': peer_target } ) except Exception as e: logging.warning(f"Due to Subscription fail to RadioStation(mother peer), " f"automatically retrying subscribe call") if response['response_code'] == message_code.Response.success: if TimerService.TIMER_KEY_SUBSCRIBE in self.__timer_service.timer_list.keys(): self.__timer_service.stop_timer(TimerService.TIMER_KEY_SUBSCRIBE) self.radio_station_stub.update_methods_version() logging.debug(f"Subscription to RadioStation(mother peer) is successful.") if TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE in self.__timer_service.timer_list.keys(): self.__timer_service.stop_timer(TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE) # start next get_status timer timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key not in self.__timer_service.timer_list.keys(): util.logger.spam(f"add timer for check_block_height_call to radiostation...") self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.GET_LAST_BLOCK_TIMER, is_repeat=True, callback=self.__check_block_height_call_to_rs_stub, callback_kwargs={"rs_rest_stub": rs_rest_stub} ) ) else: timer_key = TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE if timer_key not in self.__timer_service.timer_list.keys(): error = f"Shutdown by Subscribe retry timeout({conf.SHUTDOWN_TIMER})" self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.SHUTDOWN_TIMER, callback=self.__shutdown_peer, callback_kwargs={"message": error} ) ) return response
def __check_block_height_call_to_rs_stub(self, **kwargs): rs_rest_stub = kwargs.get("rs_rest_stub", None) response = dict() try: response = rs_rest_stub.call("GetLastBlock") except Exception as e: response['response_code'] = message_code.Response.fail if response['response_code'] == message_code.Response.success: if response['block']['height'] <= self.__block_manager.get_blockchain().block_height: # keep get last block timer, citizen subscription is still valid. return # citizen needs additional block or failed to connect to mother peer. timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key in self.__timer_service.timer_list.keys(): util.logger.spam(f"stop timer for check_block_height_call to radiostation...") self.__timer_service.stop_timer(timer_key) timer_key = TimerService.TIMER_KEY_SUBSCRIBE if timer_key not in self.__timer_service.timer_list.keys(): self.__timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.SUBSCRIBE_RETRY_TIMER, is_repeat=True, callback=self.__subscribe_call_to_rs_stub, callback_kwargs={"rs_rest_stub": rs_rest_stub} ) )
def register_peers(self): util.logger.spam(f"register_peers() : start register to peer_manager") logging.debug( f"register_peers() : channel_list = {self.admin_manager.get_channel_list()}" ) for channel_name, channel_data in self.admin_manager.json_data.items(): peer_manager = self.channel_manager.get_peer_manager(channel_name) for peer_data in channel_data['peers']: peer_info = { "id": peer_data['id'], "peer_target": peer_data['peer_target'], "order": peer_data['order'] } logging.debug( f"register Peer : channel = {channel_name}, peer_info = {peer_info}" ) peer_manager.add_peer(peer_info) if conf.ENABLE_RADIOSTATION_HEARTBEAT: timer_key = f"{TimerService.TIMER_KEY_RS_HEARTBEAT}_{channel_name}" if timer_key not in self.timer_service.timer_list: self.timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf. SLEEP_SECONDS_IN_RADIOSTATION_HEARTBEAT, is_repeat=True, callback=self.check_peer_status, callback_kwargs={"channel": channel_name}))
def start_subscribe_timer(self): timer_key = TimerService.TIMER_KEY_SUBSCRIBE if timer_key not in self.__timer_service.timer_list: self.__timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.SUBSCRIBE_RETRY_TIMER, is_repeat=True, callback=self.subscribe_network))
def start(self, is_run_at_start=True): self.is_running = True self.__timer_service.add_timer( self.__timer_key, Timer(target=self.__timer_key, duration=self.__duration, is_repeat=True, is_run_at_start=is_run_at_start, callback=self.__timer_callback))
def start_shutdown_timer(self): timer_key = TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE if timer_key not in self.__timer_service.timer_list: error = f"Shutdown by Subscribe retry timeout({conf.SHUTDOWN_TIMER} sec)" self.__timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.SHUTDOWN_TIMER, callback=self.shutdown_peer, callback_kwargs={"message": error}))
def __start_broadcast_send_unconfirmed_block_timer(broadcast_func): timer_key = TimerService.TIMER_KEY_BROADCAST_SEND_UNCONFIRMED_BLOCK timer_service = ObjectManager().channel_service.timer_service timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.INTERVAL_BROADCAST_SEND_UNCONFIRMED_BLOCK, is_repeat=True, is_run_at_start=True, callback=broadcast_func))
def start_timer(self, callback): timer_key = f"{ChannelProperty().peer_id}:{self.__precommit_block.height}" logging.debug( f"start_timer ({timer_key}/{self.__precommit_block.block_hash})") timer = Timer(target=timer_key, duration=conf.TIMEOUT_FOR_PEER_VOTE, callback=callback, callback_kwargs={"epoch": self.__epoch}) ObjectManager().channel_service.timer_service.add_timer( timer_key, timer)
def __do_vote(self): """Announce 받은 unconfirmed block 에 투표를 한다. """ if not self.__unconfirmedBlockQueue.empty(): unconfirmed_block = self.__unconfirmedBlockQueue.get() logging.debug("we got unconfirmed block ....") else: time.sleep(conf.SLEEP_SECONDS_IN_SERVICE_LOOP) # logging.debug("No unconfirmed block ....") return logging.info("PeerService received unconfirmed block: " + unconfirmed_block.block_hash) if unconfirmed_block.confirmed_transaction_list.__len__() == 0 and \ unconfirmed_block.block_type is not BlockType.peer_list: # siever 에서 사용하는 vote block 은 tx 가 없다. (검증 및 투표 불필요) # siever 에서 vote 블럭 발송 빈도를 보기 위해 warning 으로 로그 남김, 그 외의 경우 아래 로그는 주석처리 할 것 # logging.warning("This is vote block by siever") pass else: # block 검증 block_is_validated = False try: block_is_validated = Block.validate(unconfirmed_block, self.__txQueue) except Exception as e: logging.error(e) if block_is_validated: # broadcast 를 받으면 받은 블럭을 검증한 후 검증되면 자신의 blockchain 의 unconfirmed block 으로 등록해 둔다. confirmed, reason = self.__blockchain.add_unconfirm_block( unconfirmed_block) if confirmed: # block is confirmed # validated 일 때 투표 할 것이냐? confirmed 일 때 투표할 것이냐? 현재는 validate 만 체크 pass elif reason == "block_height": # Announce 되는 블럭과 자신의 height 가 다르면 Block Height Sync 를 다시 시도한다. self.block_height_sync() self.__common_service.vote_unconfirmed_block( unconfirmed_block.block_hash, block_is_validated, self.__channel_name) if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft: # turn on timer when peer type is general after vote # TODO: set appropriate callback function and parameters timer = Timer( unconfirmed_block.block_hash, conf.TIMEOUT_FOR_PEER_VOTE, ObjectManager().peer_service.timer_test_callback_function, ["test after vote by block_manager"]) ObjectManager().peer_service.timer_service.add_timer( unconfirmed_block.block_hash, timer)
def start_check_last_block_rs_timer(self): timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key not in self.__timer_service.timer_list: util.logger.spam( f"add timer for check_block_height_call to radiostation...") self.__timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.GET_LAST_BLOCK_TIMER, is_repeat=True, callback=self.__check_last_block_to_rs))
def _start_consensus_timer(self, delay): if delay < 0: delay = 0 timer_key = TimerService.TIMER_KEY_BLOCK_GENERATE timer_service = ObjectManager().channel_service.timer_service timer_service.add_timer( timer_key, Timer(target=timer_key, duration=delay, is_repeat=False, callback=self.consensus))
def __start_block_height_sync_timer(self): timer_key = TimerService.TIMER_KEY_BLOCK_HEIGHT_SYNC timer_service: TimerService = self.__channel_service.timer_service if timer_key not in timer_service.timer_list: util.logger.spam( f"add timer for block_request_call to radiostation...") timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.GET_LAST_BLOCK_TIMER, callback=self.block_height_sync))
def __send_tx_in_timer(self, tx_item=None): duration = 0 if tx_item: self.tx_messages_queue.append(tx_item) duration = conf.SEND_TX_LIST_DURATION if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list: self.__timer_service.add_timer( TimerService.TIMER_KEY_ADD_TX, Timer(target=TimerService.TIMER_KEY_ADD_TX, duration=duration, callback=self.__send_tx_by_timer, callback_kwargs={}))
def __send_tx_in_timer(self, tx_item=None): # util.logger.spam(f"broadcast_scheduler:__send_tx_in_timer") duration = 0 if tx_item: self.stored_tx.put(tx_item) duration = conf.SEND_TX_LIST_DURATION if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list: self.__timer_service.add_timer( TimerService.TIMER_KEY_ADD_TX, Timer(target=TimerService.TIMER_KEY_ADD_TX, duration=duration, callback=self.__send_tx_by_timer, callback_kwargs={})) else: pass
async def node_ws_PublishHeartbeat(self, **kwargs): def _callback(exception): self._exception = exception timer_key = TimerService.TIMER_KEY_WS_HEARTBEAT timer_service = ObjectManager().channel_service.timer_service if timer_key in timer_service.timer_list: timer_service.reset_timer(timer_key) else: timer = Timer(target=timer_key, duration=3 * conf.TIMEOUT_FOR_WS_HEARTBEAT, callback=_callback, callback_kwargs={ 'exception': ConnectionError("No Heartbeat.") }) timer_service.add_timer(timer_key, timer)
def __init__(self, timer_key, duration, timer_service: TimerService, callback, callback_lock): self.__slot = 0 self.__delayed = True self.__timer_key = timer_key self.__timer_service = timer_service self.__callback = callback self.__callback_lock = callback_lock timer_service.add_timer( timer_key, Timer(target=timer_key, duration=duration, is_repeat=True, is_run_at_start=True, callback=self.__timer_callback))
def connect_to_radio_station(self, is_reconnect=False): response = self.__radio_station_stub.call_in_times( method_name="ConnectPeer", message=loopchain_pb2.ConnectPeerRequest( channel=ChannelProperty().name, peer_object=b'', peer_id=ChannelProperty().peer_id, peer_target=ChannelProperty().peer_target, group_id=ChannelProperty().group_id, cert=self.peer_auth.peer_cert), retry_times=conf.CONNECTION_RETRY_TIMES_TO_RS, is_stub_reuse=True, timeout=conf.CONNECTION_TIMEOUT_TO_RS) # start next ConnectPeer timer if TimerService.TIMER_KEY_CONNECT_PEER not in self.__timer_service.timer_list.keys( ): self.__timer_service.add_timer( TimerService.TIMER_KEY_CONNECT_PEER, Timer(target=TimerService.TIMER_KEY_CONNECT_PEER, duration=conf.CONNECTION_RETRY_TIMER, callback=self.connect_to_radio_station, callback_kwargs={"is_reconnect": True})) if is_reconnect: return if response and response.status == message_code.Response.success: peer_list_data = pickle.loads(response.peer_list) self.__peer_manager.load(peer_list_data, False) peers, peer_list = self.__peer_manager.get_peers_for_debug() logging.debug("peer list update: " + peers) # add connected peer to processes audience for each_peer in peer_list: util.logger.spam( f"peer_service:connect_to_radio_station peer({each_peer.target}-{each_peer.status})" ) if each_peer.status == PeerStatus.connected: self.__broadcast_scheduler.schedule_job( BroadcastCommand.SUBSCRIBE, each_peer.target)
async def __check_block_height_call_to_rs_stub(self, **kwargs): rs_rest_stub = kwargs.get("rs_rest_stub", None) last_block = await rs_rest_stub.call_async("GetLastBlock") if last_block['height'] <= self.__block_manager.get_blockchain( ).block_height: return timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION if timer_key in self.__timer_service.timer_list: util.logger.spam( f"stop timer for check_block_height_call to radiostation...") self.__timer_service.stop_timer(timer_key) timer_key = TimerService.TIMER_KEY_SUBSCRIBE if timer_key not in self.__timer_service.timer_list: self.__timer_service.add_timer( timer_key, Timer(target=timer_key, duration=conf.SUBSCRIBE_RETRY_TIMER, is_repeat=True, callback=self.__subscribe_call_from_citizen, callback_kwargs={"rs_rest_stub": rs_rest_stub}))
def ConnectPeer(self, request: loopchain_pb2.ConnectPeerRequest, context): """RadioStation 에 접속한다. 응답으로 기존의 접속된 Peer 목록을 받는다. :param request: PeerRequest :param context: :return: ConnectPeerReply """ logging.info(f"Trying to connect peer: {request.peer_id}") if conf.ENABLE_RADIOSTATION_HEARTBEAT: timer_key = f"{TimerService.TIMER_KEY_RS_HEARTBEAT}_{request.channel}" if timer_key not in ObjectManager( ).rs_service.timer_service.timer_list.keys(): ObjectManager().rs_service.timer_service.add_timer( timer_key, Timer( target=timer_key, duration=conf.SLEEP_SECONDS_IN_RADIOSTATION_HEARTBEAT, is_repeat=True, callback=ObjectManager().rs_service.check_peer_status, callback_kwargs={"channel": request.channel})) if conf.ENABLE_CHANNEL_AUTH: if request.peer_target not in ObjectManager( ).rs_service.admin_manager.get_peer_list_by_channel( request.channel): status, reason = message_code.get_response( message_code.Response.fail_invalid_peer_target) return loopchain_pb2.ConnectPeerReply(status=status, peer_list=b'', more_info=reason) channel_name = conf.LOOPCHAIN_DEFAULT_CHANNEL if not request.channel else request.channel logging.debug(f"ConnectPeer channel_name({channel_name})") logging.debug(f"Connect Peer " f"\nPeer_id : {request.peer_id}" f"\nPeer_target : {request.peer_target}" f"\nChannel : {request.channel}") peer = PeerInfo(request.peer_id, request.group_id, request.peer_target, PeerStatus.unknown, cert=request.cert) util.logger.spam(f"service::ConnectPeer try add_peer") # when first peer ConnectPeer to RS, # RS need rebuild peer list from db. # For prevent leader split by RS. with self.__load_peer_manager_lock: peer_manager = ObjectManager().rs_service.channel_manager.\ get_peer_manager(channel_name) util.logger.spam(f"before load peer_manager " f"peer_count({peer_manager.get_peer_count()})") if peer_manager.get_peer_count() == 0: util.logger.spam(f"try load peer_manager from db") peer_manager = ObjectManager().rs_service.admin_manager.\ load_peer_manager(channel_name) ObjectManager().rs_service.channel_manager.\ set_peer_manager(channel_name, peer_manager) util.logger.spam(f"after load peer_manager " f"peer_count({peer_manager.get_peer_count()})") peer_order = peer_manager.add_peer(peer) peer_list_dump = b'' status, reason = message_code.get_response(message_code.Response.fail) if peer_order > 0: try: peer_list_dump = peer_manager.dump() status, reason = message_code.get_response( message_code.Response.success) except pickle.PicklingError as e: logging.warning("fail peer_list dump") reason += " " + str(e) # save current peer_manager after ConnectPeer from new peer. ObjectManager().rs_service.admin_manager.save_peer_manager( channel_name, peer_manager) return loopchain_pb2.ConnectPeerReply(status=status, peer_list=peer_list_dump, channels=None, more_info=reason)