def _get_serve_deployment_handle(
self,
deployment: Deployment,
bound_other_args_to_resolve: Dict[str, Any],
) -> Union[RayServeHandle, RayServeSyncHandle]:
"""
Return a sync or async handle of the encapsulated Deployment based on
config.
Args:
deployment (Deployment): Deployment instance wrapped in the DAGNode.
bound_other_args_to_resolve (Dict[str, Any]): Contains args used
to configure DeploymentNode.
Returns:
RayServeHandle: Default and catch-all is to return sync handle.
return async handle only if user explicitly set
USE_SYNC_HANDLE_KEY with value of False.
"""
# TODO (jiaodong): Support configurable async handle
if USE_SYNC_HANDLE_KEY not in bound_other_args_to_resolve:
# Return sync RayServeLazySyncHandle
return RayServeLazySyncHandle(deployment.name)
elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is True:
# Return sync RayServeSyncHandle
return deployment.get_handle(sync=True)
elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is False:
# Return async RayServeHandle
return deployment.get_handle(sync=False)
else:
raise ValueError(
f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value."
)
def _configure_runtime_env(deployment: Deployment, updates: Dict):
"""Overwrites deployment's runtime_env with fields in updates.
Any fields in deployment's runtime_env that aren't in updates stay the
same.
"""
if deployment.ray_actor_options is None:
deployment._ray_actor_options = {"runtime_env": updates}
else:
current_env = deployment.ray_actor_options.get("runtime_env", {})
updates.update(current_env)
deployment.ray_actor_options["runtime_env"] = updates
def test_invalid_use_sync_handle():
deployment = Deployment(
Actor,
"test",
DeploymentConfig(),
_internal=True,
)
with pytest.raises(
ValueError,
match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value",
):
_ = DeploymentNode(
Actor,
"test",
[],
{},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: {"options_a": "hii"}},
)
with pytest.raises(
ValueError,
match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value",
):
_ = DeploymentMethodNode(
deployment,
"method",
[],
{},
{},
other_args_to_resolve={
USE_SYNC_HANDLE_KEY: None,
},
)
def __init__(
self,
# For serve structured deployment, deployment body can be import path
# to the class or function instead.
func_or_class: Union[Callable, str],
deployment_name: str,
deployment_init_args: Tuple[Any],
deployment_init_kwargs: Dict[str, Any],
ray_actor_options: Dict[str, Any],
other_args_to_resolve: Optional[Dict[str, Any]] = None,
):
# Assign instance variables in base class constructor.
super().__init__(
deployment_init_args,
deployment_init_kwargs,
ray_actor_options,
other_args_to_resolve=other_args_to_resolve,
)
if self._contains_input_node():
raise ValueError(
"InputNode handles user dynamic input the the DAG, and "
"cannot be used as args, kwargs, or other_args_to_resolve "
"in the DeploymentNode constructor because it is not available "
"at class construction or binding time.")
# Deployment can be passed into other DAGNodes as init args. This is
# supported pattern in ray DAG that user can instantiate and pass class
# instances as init args to others.
# However in ray serve we send init args via .remote() that requires
# pickling, and all DAGNode types are not picklable by design.
# Thus we need convert all DeploymentNode used in init args into
# deployment handles (executable and picklable) in ray serve DAG to make
# serve DAG end to end executable.
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = self.apply_functional(
[deployment_init_args, deployment_init_kwargs],
predictate_fn=lambda node: isinstance(node, (
DeploymentNode, DeploymentMethodNode)),
apply_fn=lambda node: node._get_serve_deployment_handle(
node._deployment, node._bound_other_args_to_resolve),
)
self._deployment: Deployment = Deployment(
func_or_class,
deployment_name,
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
ray_actor_options=ray_actor_options,
_internal=True,
)
self._deployment_handle: Union[
RayServeHandle,
RayServeSyncHandle] = self._get_serve_deployment_handle(
self._deployment, other_args_to_resolve)
def __init__(
self,
func_body: Union[Callable, str],
deployment_name,
func_args,
func_kwargs,
func_options,
other_args_to_resolve=None,
):
self._body = func_body
self._deployment_name = deployment_name
super().__init__(
func_args,
func_kwargs,
func_options,
other_args_to_resolve=other_args_to_resolve,
)
if "deployment_schema" in self._bound_other_args_to_resolve:
deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[
"deployment_schema"]
deployment_shell = schema_to_deployment(deployment_schema)
# Prefer user specified name to override the generated one.
if (inspect.isfunction(func_body)
and deployment_shell.name != func_body.__name__):
self._deployment_name = deployment_shell.name
# Set the route prefix, prefer the one user supplied,
# otherwise set it to /deployment_name
if (deployment_shell.route_prefix is None
or deployment_shell.route_prefix !=
f"/{deployment_shell.name}"):
route_prefix = deployment_shell.route_prefix
else:
route_prefix = f"/{deployment_name}"
self._deployment = deployment_shell.options(
func_or_class=func_body,
name=self._deployment_name,
init_args=(),
init_kwargs=dict(),
route_prefix=route_prefix,
)
else:
self._deployment: Deployment = Deployment(
func_body,
deployment_name,
DeploymentConfig(),
init_args=tuple(),
init_kwargs=dict(),
ray_actor_options=func_options,
_internal=True,
)
# TODO (jiaodong): Polish with async handle support later
self._deployment_handle = RayServeLazySyncHandle(deployment_name)
def from_json(cls, input_json, object_hook=None):
assert input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__
args_dict = super().from_json_base(input_json, object_hook=object_hook)
return cls(
Deployment(
input_json["import_path"],
input_json["deployment_name"],
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=args_dict["args"],
init_kwargs=args_dict["kwargs"],
ray_actor_options=args_dict["options"],
_internal=True,
),
input_json["deployment_method_name"],
args_dict["args"],
args_dict["kwargs"],
args_dict["options"],
other_args_to_resolve=args_dict["other_args_to_resolve"],
)
def __init__(
self,
# For serve structured deployment, deployment body can be import path
# to the class or function instead.
func_or_class: Union[Callable, str],
deployment_name: str,
deployment_init_args: Tuple[Any],
deployment_init_kwargs: Dict[str, Any],
ray_actor_options: Dict[str, Any],
other_args_to_resolve: Optional[Dict[str, Any]] = None,
):
# Assign instance variables in base class constructor.
super().__init__(
deployment_init_args,
deployment_init_kwargs,
ray_actor_options,
other_args_to_resolve=other_args_to_resolve,
)
if self._contains_input_node():
raise ValueError(
"InputNode handles user dynamic input the the DAG, and "
"cannot be used as args, kwargs, or other_args_to_resolve "
"in the DeploymentNode constructor because it is not available "
"at class construction or binding time.")
# Deployment can be passed into other DAGNodes as init args. This is
# supported pattern in ray DAG that user can instantiate and pass class
# instances as init args to others.
# However in ray serve we send init args via .remote() that requires
# pickling, and all DAGNode types are not picklable by design.
# Thus we need convert all DeploymentNode used in init args into
# deployment handles (executable and picklable) in ray serve DAG to make
# serve DAG end to end executable.
def replace_with_handle(node):
if isinstance(node, DeploymentNode):
return node._get_serve_deployment_handle(
node._deployment, node._bound_other_args_to_resolve)
elif isinstance(node,
(DeploymentMethodNode, DeploymentFunctionNode)):
from ray.serve.pipeline.json_serde import DAGNodeEncoder
serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder)
return RayServeDAGHandle(serve_dag_root_json)
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = self.apply_functional(
[deployment_init_args, deployment_init_kwargs],
predictate_fn=lambda node: isinstance(node, (
DeploymentNode, DeploymentMethodNode, DeploymentFunctionNode)),
apply_fn=replace_with_handle,
)
if "deployment_schema" in self._bound_other_args_to_resolve:
deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[
"deployment_schema"]
deployment_shell = schema_to_deployment(deployment_schema)
# Prefer user specified name to override the generated one.
if (inspect.isclass(func_or_class)
and deployment_shell.name != func_or_class.__name__):
deployment_name = deployment_shell.name
# Set the route prefix, prefer the one user supplied,
# otherwise set it to /deployment_name
if (deployment_shell.route_prefix is None
or deployment_shell.route_prefix !=
f"/{deployment_shell.name}"):
route_prefix = deployment_shell.route_prefix
else:
route_prefix = f"/{deployment_name}"
self._deployment = deployment_shell.options(
func_or_class=func_or_class,
name=deployment_name,
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
route_prefix=route_prefix,
)
else:
self._deployment: Deployment = Deployment(
func_or_class,
deployment_name,
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
ray_actor_options=ray_actor_options,
_internal=True,
)
self._deployment_handle: Union[
RayServeLazySyncHandle, RayServeHandle,
RayServeSyncHandle] = self._get_serve_deployment_handle(
self._deployment, other_args_to_resolve)