def __subscribe_call_to_rs_stub(self, rs_rest_stub):
response = {'response_code': message_code.Response.fail,
'message': message_code.get_response_msg(message_code.Response.fail)}
try:
if conf.REST_SSL_TYPE == conf.SSLAuthType.none:
peer_target = ChannelProperty().rest_target
else:
peer_target = f"https://{ChannelProperty().rest_target}"
response = rs_rest_stub.call(
"Subscribe", {
'channel': ChannelProperty().name,
'peer_target': peer_target
}
)
except Exception as e:
logging.warning(f"Due to Subscription fail to RadioStation(mother peer), "
f"automatically retrying subscribe call")
if response['response_code'] == message_code.Response.success:
if TimerService.TIMER_KEY_SUBSCRIBE in self.__timer_service.timer_list.keys():
self.__timer_service.stop_timer(TimerService.TIMER_KEY_SUBSCRIBE)
self.radio_station_stub.update_methods_version()
logging.debug(f"Subscription to RadioStation(mother peer) is successful.")
if TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE in self.__timer_service.timer_list.keys():
self.__timer_service.stop_timer(TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE)
# start next get_status timer
timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION
if timer_key not in self.__timer_service.timer_list.keys():
util.logger.spam(f"add timer for check_block_height_call to radiostation...")
self.__timer_service.add_timer(
timer_key,
Timer(
target=timer_key,
duration=conf.GET_LAST_BLOCK_TIMER,
is_repeat=True,
callback=self.__check_block_height_call_to_rs_stub,
callback_kwargs={"rs_rest_stub": rs_rest_stub}
)
)
else:
timer_key = TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE
if timer_key not in self.__timer_service.timer_list.keys():
error = f"Shutdown by Subscribe retry timeout({conf.SHUTDOWN_TIMER})"
self.__timer_service.add_timer(
timer_key,
Timer(
target=timer_key,
duration=conf.SHUTDOWN_TIMER,
callback=self.__shutdown_peer,
callback_kwargs={"message": error}
)
)
return response
def __check_block_height_call_to_rs_stub(self, **kwargs):
rs_rest_stub = kwargs.get("rs_rest_stub", None)
response = dict()
try:
response = rs_rest_stub.call("GetLastBlock")
except Exception as e:
response['response_code'] = message_code.Response.fail
if response['response_code'] == message_code.Response.success:
if response['block']['height'] <= self.__block_manager.get_blockchain().block_height:
# keep get last block timer, citizen subscription is still valid.
return
# citizen needs additional block or failed to connect to mother peer.
timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION
if timer_key in self.__timer_service.timer_list.keys():
util.logger.spam(f"stop timer for check_block_height_call to radiostation...")
self.__timer_service.stop_timer(timer_key)
timer_key = TimerService.TIMER_KEY_SUBSCRIBE
if timer_key not in self.__timer_service.timer_list.keys():
self.__timer_service.add_timer(
timer_key,
Timer(
target=timer_key,
duration=conf.SUBSCRIBE_RETRY_TIMER,
is_repeat=True,
callback=self.__subscribe_call_to_rs_stub,
callback_kwargs={"rs_rest_stub": rs_rest_stub}
)
)
def register_peers(self):
util.logger.spam(f"register_peers() : start register to peer_manager")
logging.debug(
f"register_peers() : channel_list = {self.admin_manager.get_channel_list()}"
)
for channel_name, channel_data in self.admin_manager.json_data.items():
peer_manager = self.channel_manager.get_peer_manager(channel_name)
for peer_data in channel_data['peers']:
peer_info = {
"id": peer_data['id'],
"peer_target": peer_data['peer_target'],
"order": peer_data['order']
}
logging.debug(
f"register Peer : channel = {channel_name}, peer_info = {peer_info}"
)
peer_manager.add_peer(peer_info)
if conf.ENABLE_RADIOSTATION_HEARTBEAT:
timer_key = f"{TimerService.TIMER_KEY_RS_HEARTBEAT}_{channel_name}"
if timer_key not in self.timer_service.timer_list:
self.timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.
SLEEP_SECONDS_IN_RADIOSTATION_HEARTBEAT,
is_repeat=True,
callback=self.check_peer_status,
callback_kwargs={"channel": channel_name}))
def start_subscribe_timer(self):
timer_key = TimerService.TIMER_KEY_SUBSCRIBE
if timer_key not in self.__timer_service.timer_list:
self.__timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.SUBSCRIBE_RETRY_TIMER,
is_repeat=True,
callback=self.subscribe_network))
def start(self, is_run_at_start=True):
self.is_running = True
self.__timer_service.add_timer(
self.__timer_key,
Timer(target=self.__timer_key,
duration=self.__duration,
is_repeat=True,
is_run_at_start=is_run_at_start,
callback=self.__timer_callback))
def start_shutdown_timer(self):
timer_key = TimerService.TIMER_KEY_SHUTDOWN_WHEN_FAIL_SUBSCRIBE
if timer_key not in self.__timer_service.timer_list:
error = f"Shutdown by Subscribe retry timeout({conf.SHUTDOWN_TIMER} sec)"
self.__timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.SHUTDOWN_TIMER,
callback=self.shutdown_peer,
callback_kwargs={"message": error}))
def __start_broadcast_send_unconfirmed_block_timer(broadcast_func):
timer_key = TimerService.TIMER_KEY_BROADCAST_SEND_UNCONFIRMED_BLOCK
timer_service = ObjectManager().channel_service.timer_service
timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.INTERVAL_BROADCAST_SEND_UNCONFIRMED_BLOCK,
is_repeat=True,
is_run_at_start=True,
callback=broadcast_func))
def start_timer(self, callback):
timer_key = f"{ChannelProperty().peer_id}:{self.__precommit_block.height}"
logging.debug(
f"start_timer ({timer_key}/{self.__precommit_block.block_hash})")
timer = Timer(target=timer_key,
duration=conf.TIMEOUT_FOR_PEER_VOTE,
callback=callback,
callback_kwargs={"epoch": self.__epoch})
ObjectManager().channel_service.timer_service.add_timer(
timer_key, timer)
def __do_vote(self):
"""Announce 받은 unconfirmed block 에 투표를 한다.
"""
if not self.__unconfirmedBlockQueue.empty():
unconfirmed_block = self.__unconfirmedBlockQueue.get()
logging.debug("we got unconfirmed block ....")
else:
time.sleep(conf.SLEEP_SECONDS_IN_SERVICE_LOOP)
# logging.debug("No unconfirmed block ....")
return
logging.info("PeerService received unconfirmed block: " +
unconfirmed_block.block_hash)
if unconfirmed_block.confirmed_transaction_list.__len__() == 0 and \
unconfirmed_block.block_type is not BlockType.peer_list:
# siever 에서 사용하는 vote block 은 tx 가 없다. (검증 및 투표 불필요)
# siever 에서 vote 블럭 발송 빈도를 보기 위해 warning 으로 로그 남김, 그 외의 경우 아래 로그는 주석처리 할 것
# logging.warning("This is vote block by siever")
pass
else:
# block 검증
block_is_validated = False
try:
block_is_validated = Block.validate(unconfirmed_block,
self.__txQueue)
except Exception as e:
logging.error(e)
if block_is_validated:
# broadcast 를 받으면 받은 블럭을 검증한 후 검증되면 자신의 blockchain 의 unconfirmed block 으로 등록해 둔다.
confirmed, reason = self.__blockchain.add_unconfirm_block(
unconfirmed_block)
if confirmed:
# block is confirmed
# validated 일 때 투표 할 것이냐? confirmed 일 때 투표할 것이냐? 현재는 validate 만 체크
pass
elif reason == "block_height":
# Announce 되는 블럭과 자신의 height 가 다르면 Block Height Sync 를 다시 시도한다.
self.block_height_sync()
self.__common_service.vote_unconfirmed_block(
unconfirmed_block.block_hash, block_is_validated,
self.__channel_name)
if conf.CONSENSUS_ALGORITHM == conf.ConsensusAlgorithm.lft:
# turn on timer when peer type is general after vote
# TODO: set appropriate callback function and parameters
timer = Timer(
unconfirmed_block.block_hash, conf.TIMEOUT_FOR_PEER_VOTE,
ObjectManager().peer_service.timer_test_callback_function,
["test after vote by block_manager"])
ObjectManager().peer_service.timer_service.add_timer(
unconfirmed_block.block_hash, timer)
def start_check_last_block_rs_timer(self):
timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION
if timer_key not in self.__timer_service.timer_list:
util.logger.spam(
f"add timer for check_block_height_call to radiostation...")
self.__timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.GET_LAST_BLOCK_TIMER,
is_repeat=True,
callback=self.__check_last_block_to_rs))
def _start_consensus_timer(self, delay):
if delay < 0:
delay = 0
timer_key = TimerService.TIMER_KEY_BLOCK_GENERATE
timer_service = ObjectManager().channel_service.timer_service
timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=delay,
is_repeat=False,
callback=self.consensus))
def __start_block_height_sync_timer(self):
timer_key = TimerService.TIMER_KEY_BLOCK_HEIGHT_SYNC
timer_service: TimerService = self.__channel_service.timer_service
if timer_key not in timer_service.timer_list:
util.logger.spam(
f"add timer for block_request_call to radiostation...")
timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.GET_LAST_BLOCK_TIMER,
callback=self.block_height_sync))
def __send_tx_in_timer(self, tx_item=None):
duration = 0
if tx_item:
self.tx_messages_queue.append(tx_item)
duration = conf.SEND_TX_LIST_DURATION
if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list:
self.__timer_service.add_timer(
TimerService.TIMER_KEY_ADD_TX,
Timer(target=TimerService.TIMER_KEY_ADD_TX,
duration=duration,
callback=self.__send_tx_by_timer,
callback_kwargs={}))
def __send_tx_in_timer(self, tx_item=None):
# util.logger.spam(f"broadcast_scheduler:__send_tx_in_timer")
duration = 0
if tx_item:
self.stored_tx.put(tx_item)
duration = conf.SEND_TX_LIST_DURATION
if TimerService.TIMER_KEY_ADD_TX not in self.__timer_service.timer_list:
self.__timer_service.add_timer(
TimerService.TIMER_KEY_ADD_TX,
Timer(target=TimerService.TIMER_KEY_ADD_TX,
duration=duration,
callback=self.__send_tx_by_timer,
callback_kwargs={}))
else:
pass
async def node_ws_PublishHeartbeat(self, **kwargs):
def _callback(exception):
self._exception = exception
timer_key = TimerService.TIMER_KEY_WS_HEARTBEAT
timer_service = ObjectManager().channel_service.timer_service
if timer_key in timer_service.timer_list:
timer_service.reset_timer(timer_key)
else:
timer = Timer(target=timer_key,
duration=3 * conf.TIMEOUT_FOR_WS_HEARTBEAT,
callback=_callback,
callback_kwargs={
'exception': ConnectionError("No Heartbeat.")
})
timer_service.add_timer(timer_key, timer)
def __init__(self, timer_key, duration, timer_service: TimerService,
callback, callback_lock):
self.__slot = 0
self.__delayed = True
self.__timer_key = timer_key
self.__timer_service = timer_service
self.__callback = callback
self.__callback_lock = callback_lock
timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=duration,
is_repeat=True,
is_run_at_start=True,
callback=self.__timer_callback))
def connect_to_radio_station(self, is_reconnect=False):
response = self.__radio_station_stub.call_in_times(
method_name="ConnectPeer",
message=loopchain_pb2.ConnectPeerRequest(
channel=ChannelProperty().name,
peer_object=b'',
peer_id=ChannelProperty().peer_id,
peer_target=ChannelProperty().peer_target,
group_id=ChannelProperty().group_id,
cert=self.peer_auth.peer_cert),
retry_times=conf.CONNECTION_RETRY_TIMES_TO_RS,
is_stub_reuse=True,
timeout=conf.CONNECTION_TIMEOUT_TO_RS)
# start next ConnectPeer timer
if TimerService.TIMER_KEY_CONNECT_PEER not in self.__timer_service.timer_list.keys(
):
self.__timer_service.add_timer(
TimerService.TIMER_KEY_CONNECT_PEER,
Timer(target=TimerService.TIMER_KEY_CONNECT_PEER,
duration=conf.CONNECTION_RETRY_TIMER,
callback=self.connect_to_radio_station,
callback_kwargs={"is_reconnect": True}))
if is_reconnect:
return
if response and response.status == message_code.Response.success:
peer_list_data = pickle.loads(response.peer_list)
self.__peer_manager.load(peer_list_data, False)
peers, peer_list = self.__peer_manager.get_peers_for_debug()
logging.debug("peer list update: " + peers)
# add connected peer to processes audience
for each_peer in peer_list:
util.logger.spam(
f"peer_service:connect_to_radio_station peer({each_peer.target}-{each_peer.status})"
)
if each_peer.status == PeerStatus.connected:
self.__broadcast_scheduler.schedule_job(
BroadcastCommand.SUBSCRIBE, each_peer.target)
async def __check_block_height_call_to_rs_stub(self, **kwargs):
rs_rest_stub = kwargs.get("rs_rest_stub", None)
last_block = await rs_rest_stub.call_async("GetLastBlock")
if last_block['height'] <= self.__block_manager.get_blockchain(
).block_height:
return
timer_key = TimerService.TIMER_KEY_GET_LAST_BLOCK_KEEP_CITIZEN_SUBSCRIPTION
if timer_key in self.__timer_service.timer_list:
util.logger.spam(
f"stop timer for check_block_height_call to radiostation...")
self.__timer_service.stop_timer(timer_key)
timer_key = TimerService.TIMER_KEY_SUBSCRIBE
if timer_key not in self.__timer_service.timer_list:
self.__timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.SUBSCRIBE_RETRY_TIMER,
is_repeat=True,
callback=self.__subscribe_call_from_citizen,
callback_kwargs={"rs_rest_stub": rs_rest_stub}))
def ConnectPeer(self, request: loopchain_pb2.ConnectPeerRequest, context):
"""RadioStation 에 접속한다. 응답으로 기존의 접속된 Peer 목록을 받는다.
:param request: PeerRequest
:param context:
:return: ConnectPeerReply
"""
logging.info(f"Trying to connect peer: {request.peer_id}")
if conf.ENABLE_RADIOSTATION_HEARTBEAT:
timer_key = f"{TimerService.TIMER_KEY_RS_HEARTBEAT}_{request.channel}"
if timer_key not in ObjectManager(
).rs_service.timer_service.timer_list.keys():
ObjectManager().rs_service.timer_service.add_timer(
timer_key,
Timer(
target=timer_key,
duration=conf.SLEEP_SECONDS_IN_RADIOSTATION_HEARTBEAT,
is_repeat=True,
callback=ObjectManager().rs_service.check_peer_status,
callback_kwargs={"channel": request.channel}))
if conf.ENABLE_CHANNEL_AUTH:
if request.peer_target not in ObjectManager(
).rs_service.admin_manager.get_peer_list_by_channel(
request.channel):
status, reason = message_code.get_response(
message_code.Response.fail_invalid_peer_target)
return loopchain_pb2.ConnectPeerReply(status=status,
peer_list=b'',
more_info=reason)
channel_name = conf.LOOPCHAIN_DEFAULT_CHANNEL if not request.channel else request.channel
logging.debug(f"ConnectPeer channel_name({channel_name})")
logging.debug(f"Connect Peer "
f"\nPeer_id : {request.peer_id}"
f"\nPeer_target : {request.peer_target}"
f"\nChannel : {request.channel}")
peer = PeerInfo(request.peer_id,
request.group_id,
request.peer_target,
PeerStatus.unknown,
cert=request.cert)
util.logger.spam(f"service::ConnectPeer try add_peer")
# when first peer ConnectPeer to RS,
# RS need rebuild peer list from db.
# For prevent leader split by RS.
with self.__load_peer_manager_lock:
peer_manager = ObjectManager().rs_service.channel_manager.\
get_peer_manager(channel_name)
util.logger.spam(f"before load peer_manager "
f"peer_count({peer_manager.get_peer_count()})")
if peer_manager.get_peer_count() == 0:
util.logger.spam(f"try load peer_manager from db")
peer_manager = ObjectManager().rs_service.admin_manager.\
load_peer_manager(channel_name)
ObjectManager().rs_service.channel_manager.\
set_peer_manager(channel_name, peer_manager)
util.logger.spam(f"after load peer_manager "
f"peer_count({peer_manager.get_peer_count()})")
peer_order = peer_manager.add_peer(peer)
peer_list_dump = b''
status, reason = message_code.get_response(message_code.Response.fail)
if peer_order > 0:
try:
peer_list_dump = peer_manager.dump()
status, reason = message_code.get_response(
message_code.Response.success)
except pickle.PicklingError as e:
logging.warning("fail peer_list dump")
reason += " " + str(e)
# save current peer_manager after ConnectPeer from new peer.
ObjectManager().rs_service.admin_manager.save_peer_manager(
channel_name, peer_manager)
return loopchain_pb2.ConnectPeerReply(status=status,
peer_list=peer_list_dump,
channels=None,
more_info=reason)