def send(self, request: Event) -> ResultStatus:
# Just check event name is correct
if len(request.name) == 0:
raise RpcException("Empty event name", ResultCode.ERROR_PARAM_MISSING)
if " " in request.name:
raise RpcException(f"Invalid event name: {request.name}", ResultCode.ERROR_PARAM_INVALID)
return self.__internal_send(request)
def __check_proxy_names(
self, request: Union[ProxyRegisterRequest,
Filter]) -> List[ServiceInfo]:
# Verify input proxy names
if len(request.names) == 0 or any(len(x) == 0 for x in request.names):
raise RpcException("Missing input name in request",
ResultCode.ERROR_PARAM_MISSING)
services = self.__check_service_names(request)
if any(not srv.is_proxy for srv in services):
raise RpcException(
"At least one of the required services is not a proxy",
ResultCode.ERROR_PARAM_INVALID)
return services
def proxy_register(self, request: ProxyRegisterRequest) -> ResultStatus:
# Verify input params
services = self.__check_proxy_names(request)
if len(request.version) == 0 or request.port == 0:
raise RpcException(
"Missing input parameter in request (one of version or port)",
ResultCode.ERROR_PARAM_MISSING)
# Ready to update service proxy info
with self.lock:
for srv in services:
# Update info
srv.version = request.version
srv.proxy_port = request.port
srv.proxy_host = request.host
# Persist proxy info
self.__persist_proxies()
# Send proxy registration event
if self.__can_send_events: # pragma: no branch
self.client.events.send(
RpcEvent(
name="RPC_PROXY_REGISTER",
properties=[
EventProperty(name="names",
value=",".join(request.names)),
EventProperty(name="version", value=request.version),
EventProperty(name="port", value=str(request.port)),
EventProperty(name="host", value=request.host),
],
))
return ResultStatus()
def __check_service_names(
self, request: Union[ProxyRegisterRequest,
Filter]) -> List[ServiceInfo]:
if any(n not in self.__info for n in request.names):
raise RpcException(
"At least one of the required service names is unknown",
ResultCode.ERROR_ITEM_UNKNOWN)
return list(map(lambda n: self.__info[n], request.names))
def interrupt(self, request: EventInterrupt) -> ResultStatus:
# Known index?
q = self.__get_queue(request.client_id)
# Refuse to interrupt again already interrupted queue
with self.lock:
if request.client_id in self.__interrupt_times and self.__interrupt_times[request.client_id] is not None:
raise RpcException(f"Already interrupted ID: {request.client_id}", ResultCode.ERROR_STATE_UNEXPECTED)
# Put "None" event (will wait until listen loop is over)
q.put(None)
return ResultStatus()
def __init__(self, manager: RpcManager, stub: object, info: ServiceInfo,
server: object):
# Filter on manager methods
logger = manager.logger if not info.is_proxy else logging.getLogger(
"RpcServer")
logger.debug(f"Initializing RPC servicer for {info.name}")
fake_stub = stub(insecure_channel("localhost:1"))
for n in filter(
lambda x: not x.startswith("_") and callable(
getattr(manager, x)) and hasattr(fake_stub, x),
dir(manager)):
method = getattr(manager, n)
sig = inspect.signature(method)
return_type = sig.return_annotation
# Only methods with declared return type + one input parameter (and we're not registering a proxy)
if return_type != inspect._empty and len(
sig.parameters) == 1 and not info.is_proxy:
if return_type == Result: # pragma: no cover
raise RpcException(
f"Can't declare {n} rpc with Result as a return type; please use Return Status instead",
ResultCode.ERROR_PARAM_INVALID)
streaming = is_streaming(fake_stub, n)
logger.debug(
f" >> add method {n} (returns {return_type}){' [streaming]' if streaming else ''}"
)
setattr(
self,
n,
RpcStreamingMethod(n, manager, None, return_type, info,
server)
if streaming else RpcSimpleMethod(
n, manager, None, return_type, info, server),
)
# Otherwise, methods are coming from the parent stub
else:
if info.is_proxy:
# Register proxy method
streaming = is_streaming(fake_stub, n)
logger.debug(
f" >> add proxy method {n}{' [streaming]' if streaming else ''}"
)
setattr(
self, n,
RpcStreamingMethod(n, None, stub, None, info, server)
if streaming else RpcSimpleMethod(
n, None, stub, None, info, server))
else:
# Register stub method (not defined in the manager - will raise an exception on runtime)
logger.debug(f" >> add stub method {n}")
setattr(self, n, method)
def __get_queue(self, index: int) -> Queue:
with self.lock:
q = self.__queues[index] if index in self.__queues else None
if q is None:
raise RpcException(f"Unknown event listening ID: {index}", ResultCode.ERROR_ITEM_UNKNOWN)
return q
def delegate_call(self, request, context):
# Verify API version
metadata = RpcMetadata.from_context(context)
client_version = None
if len(metadata.api_version):
client_version = int(metadata.api_version)
if client_version > self.info.current_api_version:
raise RpcException(
f"Server current API version ({self.info.current_api_version}) is too old for client API version ({client_version})",
rc=ResultCode.ERROR_API_SERVER_TOO_OLD,
)
elif client_version < self.info.supported_api_version:
raise RpcException(
f"Client API version ({client_version}) is too old for server supported API version ({self.info.current_api_version})",
rc=ResultCode.ERROR_API_CLIENT_TOO_OLD,
)
# Proxy?
if self.info.is_proxy:
# Delegate to remote proxy, if port is set
first_try = time.time()
while self.info.proxy_port == 0:
# Wait a bit more for proxy registration
if (time.time() -
first_try) < RpcStaticConfig.CLIENT_TIMEOUT.float_val:
# Server is not available, and timeout didn't expired yet: sleep and retry
time.sleep(0.5)
self.logger.debug(
f"Proxy not registered yet for method {self.name}: wait a bit..."
)
else:
# Timeout expired... notify the caller
raise RpcException(
"Proxy didn't registered in time for method call",
ResultCode.ERROR_PROXY_UNREGISTERED)
# Reuse metadata from context
client_meta = copy.copy(metadata)
client_meta.client = f"{client_meta.client}(proxied)"
# Temporary client to proxied server (that won't raise exceptions: let's simply forward the output message to final client)
client = RpcClient(
self.info.proxy_host if len(self.info.proxy_host) else
RpcStaticConfig.MAIN_HOST.str_val,
self.info.proxy_port,
{
"stub": (self.stub, client_version if client_version
is not None else self.info.current_api_version)
},
name=client_meta,
timeout=RpcStaticConfig.CLIENT_TIMEOUT.float_val,
logger=self.logger,
exception=False,
)
# Call remote method
result_provider = getattr(client.stub, self.name)(request)
else:
# Not a proxy method, delegate to manager
result_provider = self.manager_method(request)
return result_provider
def __init__(
self,
port: int,
descriptors: List[RpcServiceDescriptor],
folders: Folders = None,
cli_config: Dict[str, str] = None,
static_items: List[Config] = None,
user_items: List[Config] = None,
with_events: bool = False,
with_debug_signal: bool = True,
):
RpcManager.__init__(self, PROXY_FILE)
self.__port = port
self.calls = []
self.__shutdown_event = Event()
# Prepare config manager
# (by the way, add our own static items)
config_m = ConfigManager(
folders, cli_config, (static_items + [RpcStaticConfig])
if static_items is not None else [RpcStaticConfig], user_items)
# Create server instance, disabling port reuse
self.__server = server(futures.ThreadPoolExecutor(
max_workers=RpcStaticConfig.MAX_WORKERS.int_val),
options=[("grpc.so_reuseport", 0)])
# Systematically add services:
# - to handle server basic operations
# - to handle remote user configuration update
# - to handle remote loggers configuration update
# - to handle events (if required)
base_descriptors = [
RpcServiceDescriptor(grpc_helper, "srv", ServerApiVersion, self,
add_RpcServerServiceServicer_to_server,
RpcServerServiceStub),
RpcServiceDescriptor(grpc_helper, "config", ConfigApiVersion,
config_m, add_ConfigServiceServicer_to_server,
ConfigServiceStub),
RpcServiceDescriptor(grpc_helper, "log", LoggerApiVersion,
LogsManager(),
add_LoggerServiceServicer_to_server,
LoggerServiceStub),
]
if with_events:
base_descriptors.append(
RpcServiceDescriptor(grpc_helper, "events", EventApiVersion,
EventsManager(),
add_EventServiceServicer_to_server,
EventServiceStub))
self.descriptors = {d.name: d for d in base_descriptors}
# Get servicers
self.descriptors.update({d.name: d for d in descriptors})
# Load persisted proxies model
proxies_model = self._load_config(folders.workspace)
# Prepare auto clients stubs map
stubs_map = {}
# Register everything
self.__info = {}
for descriptor in self.descriptors.values():
# Prepare folders and logger (only for non-proxy)
if not descriptor.is_proxy:
descriptor.manager._init_folders_n_logger(folders, port)
# Persisted model?
persisted_model = proxies_model[
descriptor.name] if descriptor.name in proxies_model else None
# Build info for service
versions = list(descriptor.api_version.values())
versions.remove(0)
current_version = max(versions)
info = ServiceInfo(
name=descriptor.name,
version=persisted_model[ProxyModel.VERSION]
if persisted_model is not None else
f"{descriptor.module.__title__}:{descriptor.module.__version__}",
current_api_version=current_version,
supported_api_version=min(versions),
is_proxy=descriptor.is_proxy,
proxy_port=persisted_model[ProxyModel.PORT]
if persisted_model is not None else None,
proxy_host=persisted_model[ProxyModel.HOST]
if persisted_model is not None else None,
)
self.logger.debug(
f"Registering service in RPC server: {trace_buffer(info)}")
# Remember info
self.__info[info.name] = info
# Register servicer in RPC server
descriptor.register_method(
RpcServicer(descriptor.manager, descriptor.client_stub, info,
self), self.__server)
# Contribute to auto-client stubs map
stubs_map[descriptor.name] = (descriptor.client_stub,
current_version)
# Setup port and start
self.logger.debug(f"Starting RPC server on port {self.__port}")
try:
self.__server.add_insecure_port(f"[::]:{self.__port}")
except Exception as e: # NOQA: B902
msg = f"Failed to start RPC server on port {self.__port}: {e}"
self.logger.error(msg)
raise RpcException(msg, rc=ResultCode.ERROR_PORT_BUSY)
self.__server.start()
self.logger.debug(f"RPC server started on port {self.__port}")
# Hook debug signal if required
if with_debug_signal and os.name == "posix":
signal.signal(signal.SIGUSR2, self.__dump_debug)
# Load all managers (including client pointing to all served services)
to_raise = None
for descriptor in self.__real_descriptors:
client = RpcClient(
"localhost",
self.__port,
stubs_map,
name=f"{descriptor.name}-client",
timeout=RpcStaticConfig.CLIENT_TIMEOUT.float_val,
logger=descriptor.manager.logger,
)
try:
descriptor.manager._preload(client, folders)
except Exception as e:
self.logger.error(
f"Error occurred during {descriptor.name} manager loading: {e}\n"
+ "".join(traceback.format_tb(e.__traceback__)))
to_raise = e
break
# If something bad happened during managers loading, shutdown and raise exception
if to_raise is not None:
self.shutdown()
raise to_raise