def waiting_service_ready(self, timeout_seconds=60, enable_k8s=True): begin_time = time.time() request = message_pb2.HeartBeatRequest() while True: try: response = self._stub.HeartBeat(request) except Exception: # grpc disconnect is expect response = None finally: if response is not None: # connnect to coordinator, fetch log if enable_k8s: self.fetch_logs() if response.status.code == error_codes_pb2.OK: logger.info( "GraphScope coordinator service connected.") break time.sleep(1) if time.time() - begin_time >= timeout_seconds: if response is None: msg = "grpc connnect failed." else: msg = response.status.error_msg raise ConnectionError( "Connect coordinator timeout, {}".format(msg))
def HeartBeat(self, request, context): if self._request and self._request.dangling_timeout_seconds >= 0: # Reset dangling detect timer if self._dangling_detecting_timer: self._dangling_detecting_timer.cancel() self._dangling_detecting_timer = threading.Timer( interval=self._request.dangling_timeout_seconds, function=self._cleanup, args=( self._request.cleanup_instance, True, ), ) self._dangling_detecting_timer.start() # analytical engine request = message_pb2.HeartBeatRequest() try: self._analytical_engine_stub.HeartBeat(request) except Exception as e: return self._make_response( message_pb2.HeartBeatResponse, error_codes_pb2.CONNECTION_ERROR, "connect analytical engine failed: {}".format(str(e)), ) else: return self._make_response( message_pb2.HeartBeatResponse, error_codes_pb2.OK )
def waiting_service_ready(self, timeout_seconds=60): begin_time = time.time() request = message_pb2.HeartBeatRequest() # Do not drop this line, which is for handling KeyboardInterrupt. response = None while True: try: response = self._stub.HeartBeat(request) except Exception: # grpc disconnect is expect response = None finally: if response is not None: if response.status.code == error_codes_pb2.OK: logger.info( "GraphScope coordinator service connected.") break time.sleep(1) if time.time() - begin_time >= timeout_seconds: if response is None: msg = "grpc connnect failed." else: msg = response.status.error_msg raise ConnectionError( "Connect coordinator timeout, {}".format(msg))
def waiting_service_ready(self, timeout_seconds=60): begin_time = time.time() request = message_pb2.HeartBeatRequest() while True: if self._launcher: code = self._launcher.poll() if code is not None and code != 0: raise RuntimeError( f"Start coordinator failed with exit code {code}" ) try: self._stub.HeartBeat(request) logger.info("GraphScope coordinator service connected.") break except grpc.RpcError as e: # Cannot connect to coordinator for a short time is expected # as the coordinator takes some time to launch msg = f"code: {e.code().name}, details: {e.details()}" if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED: logger.warning("Heart beat analytical engine failed, %s", msg) if time.time() - begin_time >= timeout_seconds: raise ConnectionError(f"Connect coordinator timeout, {msg}") time.sleep(1)
def send_heartbeat(self): request = message_pb2.HeartBeatRequest() return self._stub.HeartBeat(request)