async def __call__(self, scope, receive, send):
http_body_bytes = await self.receive_http_body(scope, receive, send)
headers = {k.decode(): v.decode() for k, v in scope["headers"]}
if self.handle is None:
self.handle = self.client.get_handle(self.endpoint_tag, sync=False)
object_ref = await self.handle.options(
method_name=headers.get("X-SERVE-CALL-METHOD".lower(),
DEFAULT.VALUE),
shard_key=headers.get("X-SERVE-SHARD-KEY".lower(), DEFAULT.VALUE),
http_method=scope["method"].upper(),
http_headers=headers).remote(
build_starlette_request(scope, http_body_bytes))
result = await object_ref
if isinstance(result, RayTaskError):
error_message = "Task Error. Traceback: {}.".format(result)
await Response(error_message,
status_code=500).send(scope, receive, send)
elif isinstance(result, starlette.responses.Response):
await result(scope, receive, send)
else:
await Response(result).send(scope, receive, send)
def parse_request_item(request_item):
if len(request_item.args) == 1:
arg = request_item.args[0]
if request_item.metadata.http_arg_is_pickled:
assert isinstance(arg, bytes)
arg: HTTPRequestWrapper = pickle.loads(arg)
return (build_starlette_request(arg.scope, arg.body), ), {}
return request_item.args, request_item.kwargs
def parse_request_item(request_item):
arg = request_item.args[0] if len(request_item.args) == 1 else None
# If the input data from handle is web request, we don't need to wrap
# it in ServeRequest.
if isinstance(arg, starlette.requests.Request):
return arg
elif isinstance(arg, HTTPRequestWrapper):
return build_starlette_request(arg.scope, arg.body)
else:
return ServeRequest(
arg,
request_item.kwargs,
headers=request_item.metadata.http_headers,
method=request_item.metadata.http_method,
)
def parse_request_item(request_item):
if request_item.metadata.request_context == TaskContext.Web:
asgi_scope, body_bytes = request_item.args
return build_starlette_request(asgi_scope, body_bytes)
else:
arg = request_item.args[0] if len(request_item.args) == 1 else None
# If the input data from handle is web request, we don't need to wrap
# it in ServeRequest.
if isinstance(arg, starlette.requests.Request):
return arg
return ServeRequest(
arg,
request_item.kwargs,
headers=request_item.metadata.http_headers,
method=request_item.metadata.http_method,
)
async def __call__(self, scope, receive, send):
http_body_bytes = await self.receive_http_body(scope, receive, send)
headers = {k.decode(): v.decode() for k, v in scope["headers"]}
# Modify the path and root path so that reverse lookups and redirection
# work as expected. We do this here instead of in replicas so it can be
# changed without restarting the replicas.
scope["path"] = scope["path"].replace(self.path_prefix, "", 1)
scope["root_path"] = self.path_prefix
starlette_request = build_starlette_request(scope, http_body_bytes)
handle = self.handle.options(
method_name=headers.get("X-SERVE-CALL-METHOD".lower(),
DEFAULT.VALUE),
shard_key=headers.get("X-SERVE-SHARD-KEY".lower(), DEFAULT.VALUE),
http_method=scope["method"].upper(),
http_headers=headers)
retries = 0
backoff_time_s = 0.05
while retries < MAX_ACTOR_FAILURE_RETRIES:
object_ref = await handle.remote(starlette_request)
try:
result = await object_ref
break
except RayActorError:
logger.warning(
"Request failed due to actor failure. There are "
f"{MAX_ACTOR_FAILURE_RETRIES - retries} retries "
"remaining.")
await asyncio.sleep(backoff_time_s)
backoff_time_s *= 2
retries += 1
if isinstance(result, RayTaskError):
error_message = "Task Error. Traceback: {}.".format(result)
await Response(error_message,
status_code=500).send(scope, receive, send)
elif isinstance(result, starlette.responses.Response):
await result(scope, receive, send)
else:
await Response(result).send(scope, receive, send)
def parse_request_item(request_item):
if len(request_item.args) <= 1:
arg = request_item.args[0] if len(request_item.args) == 1 else None
# If the input data from handle is web request, we don't need to wrap
# it in ServeRequest.
if isinstance(arg, starlette.requests.Request):
return (arg, ), {}
elif request_item.metadata.http_arg_is_pickled:
assert isinstance(arg, bytes)
arg: HTTPRequestWrapper = pickle.loads(arg)
return (build_starlette_request(arg.scope, arg.body), ), {}
elif request_item.metadata.use_serve_request:
return (ServeRequest(
arg,
request_item.kwargs,
headers=request_item.metadata.http_headers,
method=request_item.metadata.http_method,
), ), {}
return request_item.args, request_item.kwargs
async def __call__(self, scope, receive, send):
http_body_bytes = await self.receive_http_body(scope, receive, send)
headers = {k.decode(): v.decode() for k, v in scope["headers"]}
retries = 0
backoff_time_s = 0.05
while retries < MAX_ACTOR_FAILURE_RETRIES:
object_ref = await self.handle.options(
method_name=headers.get("X-SERVE-CALL-METHOD".lower(),
DEFAULT.VALUE),
shard_key=headers.get("X-SERVE-SHARD-KEY".lower(),
DEFAULT.VALUE),
http_method=scope["method"].upper(),
http_headers=headers).remote(
build_starlette_request(scope, http_body_bytes))
try:
result = await object_ref
break
except RayActorError:
logger.warning(
"Request failed due to actor failure. There are "
f"{MAX_ACTOR_FAILURE_RETRIES - retries} retries "
"remaining.")
await asyncio.sleep(backoff_time_s)
backoff_time_s *= 2
retries += 1
if isinstance(result, RayTaskError):
error_message = "Task Error. Traceback: {}.".format(result)
await Response(
error_message, status_code=500).send(scope, receive, send)
elif isinstance(result, starlette.responses.Response):
await result(scope, receive, send)
else:
await Response(result).send(scope, receive, send)
async def __call__(self, scope, receive, send):
"""Implements the ASGI protocol.
See details at:
https://asgi.readthedocs.io/en/latest/specs/index.html.
"""
error_sender = self._make_error_sender(scope, receive, send)
assert self.route_table is not None, (
"Route table must be set via set_route_table.")
assert scope["type"] == "http"
current_path = scope["path"]
self.request_counter.record(1, tags={"route": current_path})
if current_path.startswith("/-/"):
await self._handle_system_request(scope, receive, send)
return
try:
endpoint_name, methods_allowed = self.route_table[current_path]
except KeyError:
error_message = (
"Path {} not found. "
"Please ping http://.../-/routes for routing table"
).format(current_path)
await error_sender(error_message, 404)
return
if scope["method"] not in methods_allowed:
error_message = ("Methods {} not allowed. "
"Available HTTP methods are {}.").format(
scope["method"], methods_allowed)
await error_sender(error_message, 405)
return
http_body_bytes = await self.receive_http_body(scope, receive, send)
headers = {k.decode(): v.decode() for k, v in scope["headers"]}
handle = self.client.get_handle(
endpoint_name, sync=False).options(
method_name=headers.get("X-SERVE-CALL-METHOD".lower(),
DEFAULT.VALUE),
shard_key=headers.get("X-SERVE-SHARD-KEY".lower(),
DEFAULT.VALUE),
http_method=scope["method"].upper(),
http_headers=headers)
request = build_starlette_request(scope, http_body_bytes)
object_ref = await handle.remote(request)
result = await object_ref
if isinstance(result, RayTaskError):
error_message = "Task Error. Traceback: {}.".format(result)
await error_sender(error_message, 500)
elif isinstance(result, starlette.responses.Response):
if isinstance(result, starlette.responses.StreamingResponse):
raise TypeError("Starlette StreamingResponse returned by "
f"backend for endpoint {endpoint_name}. "
"StreamingResponse is unserializable and not "
"supported by Ray Serve. Consider using "
"another Starlette response type such as "
"Response, HTMLResponse, PlainTextResponse, "
"or JSONResponse. If support for "
"StreamingResponse is desired, please let "
"the Ray team know by making a Github issue!")
await result(scope, receive, send)
else:
await Response(result).send(scope, receive, send)
