def waiting_service_ready(self, timeout_seconds=60, enable_k8s=True):
begin_time = time.time()
request = message_pb2.HeartBeatRequest()
while True:
try:
response = self._stub.HeartBeat(request)
except Exception:
# grpc disconnect is expect
response = None
finally:
if response is not None:
# connnect to coordinator, fetch log
if enable_k8s:
self.fetch_logs()
if response.status.code == error_codes_pb2.OK:
logger.info(
"GraphScope coordinator service connected.")
break
time.sleep(1)
if time.time() - begin_time >= timeout_seconds:
if response is None:
msg = "grpc connnect failed."
else:
msg = response.status.error_msg
raise ConnectionError(
"Connect coordinator timeout, {}".format(msg))
def HeartBeat(self, request, context):
if self._request and self._request.dangling_timeout_seconds >= 0:
# Reset dangling detect timer
if self._dangling_detecting_timer:
self._dangling_detecting_timer.cancel()
self._dangling_detecting_timer = threading.Timer(
interval=self._request.dangling_timeout_seconds,
function=self._cleanup,
args=(
self._request.cleanup_instance,
True,
),
)
self._dangling_detecting_timer.start()
# analytical engine
request = message_pb2.HeartBeatRequest()
try:
self._analytical_engine_stub.HeartBeat(request)
except Exception as e:
return self._make_response(
message_pb2.HeartBeatResponse,
error_codes_pb2.CONNECTION_ERROR,
"connect analytical engine failed: {}".format(str(e)),
)
else:
return self._make_response(
message_pb2.HeartBeatResponse, error_codes_pb2.OK
)
def waiting_service_ready(self, timeout_seconds=60):
begin_time = time.time()
request = message_pb2.HeartBeatRequest()
# Do not drop this line, which is for handling KeyboardInterrupt.
response = None
while True:
try:
response = self._stub.HeartBeat(request)
except Exception:
# grpc disconnect is expect
response = None
finally:
if response is not None:
if response.status.code == error_codes_pb2.OK:
logger.info(
"GraphScope coordinator service connected.")
break
time.sleep(1)
if time.time() - begin_time >= timeout_seconds:
if response is None:
msg = "grpc connnect failed."
else:
msg = response.status.error_msg
raise ConnectionError(
"Connect coordinator timeout, {}".format(msg))
def waiting_service_ready(self, timeout_seconds=60):
begin_time = time.time()
request = message_pb2.HeartBeatRequest()
while True:
if self._launcher:
code = self._launcher.poll()
if code is not None and code != 0:
raise RuntimeError(
f"Start coordinator failed with exit code {code}"
)
try:
self._stub.HeartBeat(request)
logger.info("GraphScope coordinator service connected.")
break
except grpc.RpcError as e:
# Cannot connect to coordinator for a short time is expected
# as the coordinator takes some time to launch
msg = f"code: {e.code().name}, details: {e.details()}"
if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
logger.warning("Heart beat analytical engine failed, %s", msg)
if time.time() - begin_time >= timeout_seconds:
raise ConnectionError(f"Connect coordinator timeout, {msg}")
time.sleep(1)
def send_heartbeat(self):
request = message_pb2.HeartBeatRequest()
return self._stub.HeartBeat(request)
