def run(self, event: threading.Event):
target_host = '[::]:' + str(self.__port)
GRPCHelper().add_server_port(self.outer_server, target_host)
target_host = conf.INNER_SERVER_BIND_IP + ':' + str(
self.__inner_service_port)
GRPCHelper().add_server_port(self.inner_server, target_host,
conf.SSLAuthType.none)
# Block Generator 에 subscribe 하게 되면 Block Generator 는 peer 에 channel 생성을 요청한다.
# 따라서 peer 의 gRPC 서버가 완전히 시작된 후 Block Generator 로 subscribe 요청을 하여야 한다.
event.set()
try:
logging.info(f'CommonService is running')
while self.is_run():
time.sleep(conf.SLEEP_SECONDS_IN_SERVICE_NONE)
except KeyboardInterrupt:
logging.info("Server Stop by KeyboardInterrupt")
finally:
if self.__inner_service_port is not None:
self.inner_server.stop(0)
self.outer_server.stop(0)
logging.info("Server thread Ended.")
def serve(self,
port: int = None,
event_for_init: multiprocessing.Event = None):
"""Peer(BlockGenerator Peer) to RadioStation
:param port: RadioStation Peer
:param event_for_init:
"""
if port is None:
port = conf.PORT_RADIOSTATION
stopwatch_start = timeit.default_timer()
self.__channel_manager = ChannelManager()
self.register_peers()
# TODO: Currently, some environments are failing to execute RestServiceRS without this sleep.
# This sleep fixes current node's issue but we need to fix it right way by investigating.
time.sleep(1)
if conf.ENABLE_REST_SERVICE:
self.__rest_service = RestServiceRS(int(port))
self.p2p_outer_server = GRPCHelper().start_outer_server(str(port))
self.p2p_inner_server = GRPCHelper().start_inner_server(str(port))
loopchain_pb2_grpc.add_RadioStationServicer_to_server(
self.__outer_service, self.p2p_outer_server)
loopchain_pb2_grpc.add_AdminServiceServicer_to_server(
self.__admin_service, self.p2p_inner_server)
logging.info("Start Radio Station service at port: " + str(port))
self.__timer_service.start()
stopwatch_duration = timeit.default_timer() - stopwatch_start
logging.info(
f"Start Radio Station service at port: {port} start duration({stopwatch_duration})"
)
if event_for_init is not None:
event_for_init.set()
signal.signal(signal.SIGINT, self.close)
signal.signal(signal.SIGTERM, self.close)
self.__timer_service.wait()
if self.__rest_service is not None:
self.__rest_service.stop()
def run(self, conn, event: multiprocessing.Event):
logging.debug("Container run...")
if self._type == ServerType.GRPC:
logging.info(f'Container run grpc port {self._port}')
setproctitle.setproctitle(f"{setproctitle.getproctitle()} {self._process_name}")
server = grpc.server(futures.ThreadPoolExecutor(conf.MAX_WORKERS, "ContainerThread"))
loopchain_pb2_grpc.add_ContainerServicer_to_server(self, server)
GRPCHelper().add_server_port(server, '[::]:' + str(self._port), conf.SSLAuthType.none)
logging.info(f'Container run complete grpc port {self._port}')
elif self._type == ServerType.REST_PEER:
args = ['python3', '-m', 'loopchain', 'rest', '-p', str(self._port)]
args += command_arguments.get_raw_commands_by_filter(
command_arguments.Type.AMQPTarget,
command_arguments.Type.AMQPKey,
command_arguments.Type.Develop,
command_arguments.Type.ConfigurationFilePath,
command_arguments.Type.RadioStationTarget
)
server = CommonSubprocess(args)
api_port = self._port + conf.PORT_DIFF_REST_SERVICE_CONTAINER
server.set_proctitle(f"{setproctitle.getproctitle()} RestServer api_port({api_port})")
else:
args = ['python3', '-m', 'loopchain', 'rest-rs', '-p', str(self._port)]
args += command_arguments.get_raw_commands_by_filter(
command_arguments.Type.Develop,
command_arguments.Type.ConfigurationFilePath
)
api_port = self._port + conf.PORT_DIFF_REST_SERVICE_CONTAINER
server = CommonSubprocess(args)
server.set_proctitle(f"{setproctitle.getproctitle()} RestServerRS api_port({api_port})")
logging.info(f'Container run complete port {self._port}')
# complete init
event.set()
if self._type == ServerType.GRPC:
self._append_monitor()
command = None
while command != "quit":
try:
command, param = conn.recv() # Queue 에 내용이 들어올 때까지 여기서 대기 된다. 따라서 Sleep 이 필요 없다.
logging.debug("Container got: " + str(param))
except Exception as e:
logging.warning("Container conn.recv() error: " + str(e))
except KeyboardInterrupt:
pass
if self._type == ServerType.GRPC:
server.stop(0)
else:
server.stop()
logging.info("Server Container Ended.")
def get_stub_to_server(target, stub_class, time_out_seconds=None, is_check_status=True,
ssl_auth_type: conf.SSLAuthType=conf.SSLAuthType.none):
"""gRPC connection to server
:return: stub to server
"""
if time_out_seconds is None:
time_out_seconds = conf.CONNECTION_RETRY_TIMEOUT
stub = None
channel = None
start_time = timeit.default_timer()
duration = timeit.default_timer() - start_time
while stub is None and duration < time_out_seconds:
try:
logging.debug("(util) get stub to server target: " + str(target))
channel = GRPCHelper().create_client_channel(target, ssl_auth_type, conf.GRPC_SSL_KEY_LOAD_TYPE)
stub = stub_class(channel)
if is_check_status:
stub.Request(loopchain_pb2.Message(code=message_code.Request.status), conf.GRPC_TIMEOUT)
except Exception as e:
logging.warning("Connect to Server Error(get_stub_to_server): " + str(e))
logging.debug("duration(" + str(duration)
+ ") interval(" + str(conf.CONNECTION_RETRY_INTERVAL)
+ ") timeout(" + str(time_out_seconds) + ")")
# RETRY_INTERVAL 만큼 대기후 TIMEOUT 전이면 다시 시도
time.sleep(conf.CONNECTION_RETRY_INTERVAL)
duration = timeit.default_timer() - start_time
stub = None
return stub, channel
def __get_peer_stub_list(self):
"""It updates peer list for block manager refer to peer list on the loopchain network.
This peer list is not same to the peer list of the loopchain network.
:return max_height: a height of current blockchain
:return peer_stubs: current peer list on the loopchain network
"""
max_height = -1 # current max height
unconfirmed_block_height = -1
peer_stubs = [] # peer stub list for block height synchronization
if not ObjectManager().channel_service.is_support_node_function(
conf.NodeFunction.Vote):
rest_stub = ObjectManager().channel_service.radio_station_stub
peer_stubs.append(rest_stub)
last_block = rest_stub.call("GetLastBlock")
max_height = self.__blockchain.block_versioner.get_height(
last_block)
return max_height, unconfirmed_block_height, peer_stubs
# Make Peer Stub List [peer_stub, ...] and get max_height of network
peer_target = ChannelProperty().peer_target
peer_manager = ObjectManager().channel_service.peer_manager
target_dict = peer_manager.get_IP_of_peers_dict()
target_list = [
peer_target for peer_id, peer_target in target_dict.items()
if peer_id != ChannelProperty().peer_id
]
for target in target_list:
if target != peer_target:
logging.debug(f"try to target({target})")
channel = GRPCHelper().create_client_channel(target)
stub = loopchain_pb2_grpc.PeerServiceStub(channel)
try:
response = stub.GetStatus(
loopchain_pb2.StatusRequest(
request="",
channel=self.__channel_name,
), conf.GRPC_TIMEOUT_SHORT)
response.block_height = max(
response.block_height,
response.unconfirmed_block_height)
if response.block_height > max_height:
# Add peer as higher than this
max_height = response.block_height
unconfirmed_block_height = response.unconfirmed_block_height
peer_stubs.append(stub)
except Exception as e:
logging.warning(
f"This peer has already been removed from the block height target node. {e}"
)
return max_height, unconfirmed_block_height, peer_stubs
def __get_peer_stub_list(self, target_peer_stub=None):
"""It updates peer list for block manager refer to peer list on the loopchain network.
This peer list is not same to the peer list of the loopchain network.
:return max_height: a height of current blockchain
:return peer_stubs: current peer list on the loopchain network
"""
peer_target = ChannelProperty().peer_target
peer_manager = ObjectManager().channel_service.peer_manager
# Make Peer Stub List [peer_stub, ...] and get max_height of network
max_height = -1 # current max height
peer_stubs = [] # peer stub list for block height synchronization
if ObjectManager().channel_service.is_support_node_function(
conf.NodeFunction.Vote):
target_dict = peer_manager.get_IP_of_peers_dict()
target_list = [
peer_target for peer_id, peer_target in target_dict.items()
if peer_id != ChannelProperty().peer_id
]
else:
target_list = [f"{target_peer_stub.target}"]
for target in target_list:
if target != peer_target:
logging.debug(f"try to target({target})")
channel = GRPCHelper().create_client_channel(target)
stub = loopchain_pb2_grpc.PeerServiceStub(channel)
try:
if ObjectManager(
).channel_service.is_support_node_function(
conf.NodeFunction.Vote):
response = stub.GetStatus(
loopchain_pb2.StatusRequest(
request="",
channel=self.__channel_name,
), conf.GRPC_TIMEOUT_SHORT)
else:
response = target_peer_stub.call("Status")
util.logger.spam('{/api/v1/status/peer} response: ' +
response.text)
response.block_height = int(
json.loads(response.text)["block_height"])
stub.target = target
if response.block_height > max_height:
# Add peer as higher than this
max_height = response.block_height
peer_stubs.append(stub)
except Exception as e:
logging.warning(
f"This peer has already been removed from the block height target node. {e}"
)
return max_height, peer_stubs
def get_stub_to_server(target, stub_class, ssl_auth_type: conf.SSLAuthType = conf.SSLAuthType.none):
"""gRPC connection to server
:return: stub to server
"""
stub = None
channel = None
try:
logging.debug(f"(util) get stub to server target: {target}")
channel = GRPCHelper().create_client_channel(target, ssl_auth_type, conf.GRPC_SSL_KEY_LOAD_TYPE)
stub = stub_class(channel)
except Exception as e:
logging.warning(f"Connect to Server Error(get_stub_to_server): {e}")
return stub, channel
def _get_peer_stub_list(self) -> Tuple[int, int, List[Tuple[str, Any]]]:
"""It updates peer list for block manager refer to peer list on the loopchain network.
This peer list is not same to the peer list of the loopchain network.
:return max_height: a height of current blockchain
:return unconfirmed_block_height: unconfirmed_block_height on the network
:return peer_stubs: current peer list on the network (target, peer_stub)
"""
max_height = -1 # current max height
unconfirmed_block_height = -1
peer_stubs = [] # peer stub list for block height synchronization
rs_client: RestClient = self._channel_service.rs_client
if not self._channel_service.is_support_node_function(
conf.NodeFunction.Vote):
status_response = rs_client.call(RestMethod.Status)
max_height = status_response['block_height']
peer_stubs.append((rs_client.target, rs_client))
return max_height, unconfirmed_block_height, peer_stubs
# Make Peer Stub List [peer_stub, ...] and get max_height of network
self._block_height_sync_bad_targets = {
k: v
for k, v in self._block_height_sync_bad_targets.items()
if v > self._blockchain.block_height
}
utils.logger.info(
f"Bad Block Sync Peer : {self._block_height_sync_bad_targets}")
peer_target = ChannelProperty().peer_target
my_height = self._blockchain.block_height
port_pattern = re.compile(r":([0-9]{2,5})$")
def _converter(target) -> str:
port = int(port_pattern.search(target).group(1))
new_port = f":{port + conf.PORT_DIFF_REST_SERVICE_CONTAINER}"
return port_pattern.sub(new_port, target)
endpoints = {
target: _converter(target)
for target in self._block_manager.get_target_list()
}
for grpc_endpoint, rest_endpoint in endpoints.items():
if grpc_endpoint == peer_target:
continue
if grpc_endpoint in self._block_height_sync_bad_targets:
continue
utils.logger.debug(
f"try to grpc_endpoint({grpc_endpoint}), rest_endpoint({rest_endpoint})"
)
channel = GRPCHelper().create_client_channel(grpc_endpoint)
stub = loopchain_pb2_grpc.PeerServiceStub(channel)
try:
client = RestClient(self._block_manager.channel_name,
rest_endpoint)
response: dict = client.call(RestMethod.Status,
timeout=conf.REST_TIMEOUT)
target_block_height = max(response["block_height"],
response["unconfirmed_block_height"])
recovery = response.get("recovery", {})
# only recovery_mode node should be included in block sync when running by recovery_mode
if conf.RECOVERY_MODE and not recovery.get("mode", False):
continue
if target_block_height > my_height:
peer_stubs.append((grpc_endpoint, stub))
max_height = max(max_height, target_block_height)
unconfirmed_block_height = max(
unconfirmed_block_height,
response["unconfirmed_block_height"])
except Exception as e:
utils.logger.warning(
f"This peer has already been removed from the block height target node. {e!r}"
)
return max_height, unconfirmed_block_height, peer_stubs
class RadioStationService:
"""Radiostation 의 main Class
peer 를 위한 outer service 와 관리용 admin service 두개의 gRPC interface 를 가진다.
"""
# 인증처리
__ca = None
def __init__(self,
radio_station_ip=None,
cert_path=None,
cert_pass=None,
rand_seed=None):
"""RadioStation Init
:param radio_station_ip: radioStation Ip
:param cert_path: RadioStation 인증서 디렉토리 경로
:param cert_pass: RadioStation private key password
"""
logger_preset = loggers.get_preset()
logger_preset.peer_id = "RadioStation"
logger_preset.update_logger()
if radio_station_ip is None:
radio_station_ip = conf.IP_RADIOSTATION
logging.info("Set RadioStationService IP: " + radio_station_ip)
if cert_path is not None:
logging.info("CA Certificate Path : " + cert_path)
self.__admin_manager = AdminManager("station")
self.__channel_manager = None
self.__rest_service = None
self.__timer_service = TimerService()
# RS has two status (active, standby) active means enable outer service
# standby means stop outer service and heartbeat to the other RS (active)
self.__is_active = False
# 인증 클래스
self.__ca = CertificateAuthorization()
if cert_path is not None:
# 인증서 로드
self.__ca.load_pki(cert_path, cert_pass)
logging.info("Current RadioStation SECURITY_MODE : " +
str(self.__ca.is_secure))
self.p2p_inner_server = None
self.p2p_outer_server = None
# gRPC service for Radiostation
self.__outer_service = OuterService()
self.__admin_service = AdminService(self.__admin_manager)
# {group_id:[ {peer_id:IP} ] }로 구성된 dictionary
self.peer_groups = {conf.ALL_GROUP_ID: []}
# Peer의 보안을 담당
self.auth = {}
ObjectManager().rs_service = self
def __del__(self):
pass
def launch_block_generator(self):
pass
@property
def admin_manager(self) -> AdminManager:
return self.__admin_manager
@property
def channel_manager(self) -> ChannelManager:
return self.__channel_manager
@property
def timer_service(self) -> TimerService:
return self.__timer_service
def check_peer_status(self, channel):
"""service loop for status heartbeat check to peer list
:return:
"""
util.logger.spam(
f"rs_service:check_peer_status(Heartbeat...{channel}) "
f"for reset Leader and delete no response Peer")
peer_manager = self.__channel_manager.get_peer_manager(channel)
peer_manager.check_peer_status()
def register_peers(self):
util.logger.spam(f"register_peers() : start register to peer_manager")
logging.debug(
f"register_peers() : channel_list = {self.admin_manager.get_channel_list()}"
)
for channel_name, channel_data in self.admin_manager.json_data.items():
peer_manager = self.channel_manager.get_peer_manager(channel_name)
for peer_data in channel_data['peers']:
peer_info = {
"id": peer_data['id'],
"peer_target": peer_data['peer_target'],
"order": peer_data['order']
}
logging.debug(
f"register Peer : channel = {channel_name}, peer_info = {peer_info}"
)
peer_manager.add_peer(peer_info)
if conf.ENABLE_RADIOSTATION_HEARTBEAT:
timer_key = f"{TimerService.TIMER_KEY_RS_HEARTBEAT}_{channel_name}"
if timer_key not in self.timer_service.timer_list:
self.timer_service.add_timer(
timer_key,
Timer(target=timer_key,
duration=conf.
SLEEP_SECONDS_IN_RADIOSTATION_HEARTBEAT,
is_repeat=True,
callback=self.check_peer_status,
callback_kwargs={"channel": channel_name}))
def serve(self,
port: int = None,
event_for_init: multiprocessing.Event = None):
"""Peer(BlockGenerator Peer) to RadioStation
:param port: RadioStation Peer
:param event_for_init:
"""
if port is None:
port = conf.PORT_RADIOSTATION
stopwatch_start = timeit.default_timer()
self.__channel_manager = ChannelManager()
self.register_peers()
# TODO: Currently, some environments are failing to execute RestServiceRS without this sleep.
# This sleep fixes current node's issue but we need to fix it right way by investigating.
time.sleep(1)
if conf.ENABLE_REST_SERVICE:
self.__rest_service = RestServiceRS(int(port))
self.p2p_outer_server = GRPCHelper().start_outer_server(str(port))
self.p2p_inner_server = GRPCHelper().start_inner_server(str(port))
loopchain_pb2_grpc.add_RadioStationServicer_to_server(
self.__outer_service, self.p2p_outer_server)
loopchain_pb2_grpc.add_AdminServiceServicer_to_server(
self.__admin_service, self.p2p_inner_server)
logging.info("Start Radio Station service at port: " + str(port))
self.__timer_service.start()
stopwatch_duration = timeit.default_timer() - stopwatch_start
logging.info(
f"Start Radio Station service at port: {port} start duration({stopwatch_duration})"
)
if event_for_init is not None:
event_for_init.set()
signal.signal(signal.SIGINT, self.close)
signal.signal(signal.SIGTERM, self.close)
self.__timer_service.wait()
if self.__rest_service is not None:
self.__rest_service.stop()
def close(self, sig, frame):
self.p2p_inner_server.stop(None)
self.p2p_outer_server.stop(None)
self.__timer_service.stop()
def __get_peer_stub_list(self) -> Tuple[int, int, List[Tuple]]:
"""It updates peer list for block manager refer to peer list on the loopchain network.
This peer list is not same to the peer list of the loopchain network.
:return max_height: a height of current blockchain
:return unconfirmed_block_height: unconfirmed_block_height on the network
:return peer_stubs: current peer list on the network (target, peer_stub)
"""
max_height = -1 # current max height
unconfirmed_block_height = -1
peer_stubs = [] # peer stub list for block height synchronization
if not ObjectManager().channel_service.is_support_node_function(
conf.NodeFunction.Vote):
rs_client = ObjectManager().channel_service.rs_client
status_response = rs_client.call(RestMethod.Status)
max_height = status_response['block_height']
peer_stubs.append((rs_client.target, rs_client))
return max_height, unconfirmed_block_height, peer_stubs
# Make Peer Stub List [peer_stub, ...] and get max_height of network
self.__block_height_sync_bad_targets = {
k: v
for k, v in self.__block_height_sync_bad_targets.items()
if v > self.blockchain.block_height
}
util.logger.info(
f"Bad Block Sync Peer : {self.__block_height_sync_bad_targets}")
peer_target = ChannelProperty().peer_target
my_height = self.blockchain.block_height
if self.blockchain.last_block:
reps_hash = self.blockchain.get_reps_hash_by_header(
self.blockchain.last_block.header)
else:
reps_hash = ChannelProperty().crep_root_hash
rep_targets = self.blockchain.find_preps_targets_by_roothash(reps_hash)
target_list = list(rep_targets.values())
for target in target_list:
if target == peer_target:
continue
if target in self.__block_height_sync_bad_targets:
continue
util.logger.debug(f"try to target({target})")
channel = GRPCHelper().create_client_channel(target)
stub = loopchain_pb2_grpc.PeerServiceStub(channel)
try:
response = stub.GetStatus(
loopchain_pb2.StatusRequest(
request='block_sync',
channel=self.__channel_name,
), conf.GRPC_TIMEOUT_SHORT)
target_block_height = max(response.block_height,
response.unconfirmed_block_height)
if target_block_height > my_height:
peer_stubs.append((target, stub))
max_height = max(max_height, target_block_height)
unconfirmed_block_height = max(
unconfirmed_block_height,
response.unconfirmed_block_height)
except Exception as e:
util.logger.warning(
f"This peer has already been removed from the block height target node. {e}"
)
return max_height, unconfirmed_block_height, peer_stubs
async def run_p2p_server(self):
self._p2p_outer_server = GRPCHelper().start_outer_server(
str(self._peer_port))
loopchain_pb2_grpc.add_PeerServiceServicer_to_server(
self._outer_service, self._p2p_outer_server)
await self._p2p_outer_server.start()
