def _check_shutdown(self):
assert self.lock.locked()
if not self._in_shutdown:
return
self.lock.release()
# Do not try disconnect() or throw exceptions in self.data_thread.
# Otherwise deadlock can occur.
if threading.current_thread().ident == self.data_thread.ident:
return
from ray.util import disconnect
disconnect()
self.lock.acquire()
if self._last_exception is not None:
msg = ("Request can't be sent because the Ray client has already "
"been disconnected due to an error. Last exception: "
f"{self._last_exception}")
else:
msg = ("Request can't be sent because the Ray client has already "
"been disconnected.")
raise ConnectionError(msg)
def _blocking_send(
self,
req: ray_client_pb2.DataRequest) -> ray_client_pb2.DataResponse:
if self._in_shutdown:
from ray.util import disconnect
disconnect()
raise ConnectionError(
"Request can't be sent because the data channel is "
"terminated. This is likely because the data channel "
"disconnected at some point before this request was "
"prepared. Ray Client has been disconnected.")
req_id = self._next_id()
req.req_id = req_id
self.request_queue.put(req)
data = None
with self.cv:
self.cv.wait_for(
lambda: req_id in self.ready_data or self._in_shutdown)
if self._in_shutdown:
from ray.util import disconnect
disconnect()
raise ConnectionError(
"Sending request failed because the data channel "
"terminated. This is usually due to an error "
f"in handling the most recent request: {req}. Ray Client "
"has been disconnected.")
data = self.ready_data[req_id]
del self.ready_data[req_id]
return data
def _async_send(self,
req: ray_client_pb2.DataRequest,
callback: Optional[ResponseCallable] = None) -> None:
if self._in_shutdown:
from ray.util import disconnect
disconnect()
raise ConnectionError(
"Request can't be sent because the data channel is "
"terminated. This is likely because the data channel "
"disconnected at some point before this request was "
"prepared. Ray Client has been disconnected.")
req_id = self._next_id()
req.req_id = req_id
if callback:
self.asyncio_waiting_data[req_id] = callback
self.request_queue.put(req)