def close_learning_instance(self, object_id):
if object_id not in self._learning_instance_processes:
return
# delete the services
for target in self._graphlearn_services[object_id]:
try:
delete_kubernetes_object(
api_client=self._api_client,
target=target,
wait=self._waiting_for_delete,
timeout_seconds=self._timeout_seconds,
)
except Exception as e:
logger.error("Failed to delete graphlearn service for %s, %s",
object_id, e)
# terminate the process
for proc in self._learning_instance_processes[object_id]:
try:
proc.terminate()
proc.wait(1)
except Exception as e:
logger.error("Failed to terminate graphlearn server: %s", e)
self._learning_instance_processes[object_id].clear()
def stop(self, wait=False):
"""Stop graphscope instance on kubernetes cluster.
Args:
wait: bool, optional
Waiting for delete. Defaults to False.
Raises:
TimeoutError:
Waiting for stop instance timeout when ``wait`` or ``_waiting_for_delete`` is True.
"""
if not self._closed:
# delete resources created by graphscope inside namespace
# make sure delete permission resouces in the end
for target in reversed(self._resource_object):
delete_kubernetes_object(
api_client=self._api_client,
target=target,
wait=self._waiting_for_delete,
timeout_seconds=self._timeout_seconds,
)
self._resource_object = []
if self._delete_namespace:
# delete namespace
api = CoreV1Api(self._api_client)
try:
api.delete_namespace(self._namespace)
except K8SApiException:
# namespace already deleted.
pass
else:
if wait or self._waiting_for_delete:
start_time = time.time()
while True:
try:
api.read_namespace(self._namespace)
except K8SApiException as ex:
if ex.status != 404:
raise
break
else:
time.sleep(1)
if (
self._timeout_seconds
and time.time() - start_time > self._timeout_seconds
):
logger.info(
"Deleting namespace %s timeout"
% self._namespace
)
break
self._closed = True
def stop(self, is_dangling=False):
if not self._closed:
for target in self._resource_object:
delete_kubernetes_object(
api_client=self._api_client,
target=target,
wait=self._saved_locals["waiting_for_delete"],
timeout_seconds=self._saved_locals["timeout_seconds"],
)
self._resource_object = []
if is_dangling:
logger.info("Dangling coordinator detected, cleaning up...")
# delete everything inside namespace of graphscope instance
if self._saved_locals["delete_namespace"]:
# delete namespace created by graphscope
self._core_api.delete_namespace(
self._saved_locals["namespace"])
if self._saved_locals["waiting_for_delete"]:
start_time = time.time()
while True:
try:
self._core_api.read_namespace(
self._saved_locals["namespace"])
except K8SApiException as ex:
if ex.status != 404:
logger.error(
"Deleting dangling namespace {} failed: {}"
.format(
self._saved_locals["namespace"],
str(ex)))
break
else:
time.sleep(1)
if (time.time() - start_time >
self._saved_locals["timeout_seconds"]):
logger.error(
"Deleting namespace %s timeout" %
self._saved_locals["namespace"])
else:
# delete coordinator deployment and service
self._delete_dangling_coordinator()
self._closed = True
