def _get_serve_deployment_handle(
self,
deployment: Deployment,
bound_other_args_to_resolve: Dict[str, Any],
) -> Union[RayServeHandle, RayServeSyncHandle]:
"""
Return a sync or async handle of the encapsulated Deployment based on
config.
Args:
deployment (Deployment): Deployment instance wrapped in the DAGNode.
bound_other_args_to_resolve (Dict[str, Any]): Contains args used
to configure DeploymentNode.
Returns:
RayServeHandle: Default and catch-all is to return sync handle.
return async handle only if user explicitly set
USE_SYNC_HANDLE_KEY with value of False.
"""
# TODO (jiaodong): Support configurable async handle
if USE_SYNC_HANDLE_KEY not in bound_other_args_to_resolve:
# Return sync RayServeLazySyncHandle
return RayServeLazySyncHandle(deployment.name)
elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is True:
# Return sync RayServeSyncHandle
return deployment.get_handle(sync=True)
elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is False:
# Return async RayServeHandle
return deployment.get_handle(sync=False)
else:
raise ValueError(
f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value."
)
def test_invalid_use_sync_handle():
deployment = Deployment(
Actor,
"test",
DeploymentConfig(),
_internal=True,
)
with pytest.raises(
ValueError,
match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value",
):
_ = DeploymentNode(
Actor,
"test",
[],
{},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: {"options_a": "hii"}},
)
with pytest.raises(
ValueError,
match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value",
):
_ = DeploymentMethodNode(
deployment,
"method",
[],
{},
{},
other_args_to_resolve={
USE_SYNC_HANDLE_KEY: None,
},
)
def __init__(
self,
func_body: Union[Callable, str],
deployment_name,
func_args,
func_kwargs,
func_options,
other_args_to_resolve=None,
):
self._body = func_body
self._deployment_name = deployment_name
super().__init__(
func_args,
func_kwargs,
func_options,
other_args_to_resolve=other_args_to_resolve,
)
if "deployment_schema" in self._bound_other_args_to_resolve:
deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[
"deployment_schema"
]
deployment_shell = schema_to_deployment(deployment_schema)
# Prefer user specified name to override the generated one.
if (
inspect.isfunction(func_body)
and deployment_shell.name != func_body.__name__
):
self._deployment_name = deployment_shell.name
# Set the route prefix, prefer the one user supplied,
# otherwise set it to /deployment_name
if (
deployment_shell.route_prefix is None
or deployment_shell.route_prefix != f"/{deployment_shell.name}"
):
route_prefix = deployment_shell.route_prefix
else:
route_prefix = f"/{deployment_name}"
self._deployment = deployment_shell.options(
func_or_class=func_body,
name=self._deployment_name,
init_args=(),
init_kwargs={},
route_prefix=route_prefix,
)
else:
self._deployment: Deployment = Deployment(
func_body,
deployment_name,
DeploymentConfig(),
init_args=tuple(),
init_kwargs=dict(),
ray_actor_options=func_options,
_internal=True,
)
# TODO (jiaodong): Polish with async handle support later
self._deployment_handle = RayServeLazySyncHandle(self._deployment.name)
def decorator(_func_or_class):
return Deployment(
_func_or_class,
name if name is not None else _func_or_class.__name__,
config,
version=version,
init_args=init_args,
init_kwargs=init_kwargs,
route_prefix=route_prefix,
ray_actor_options=ray_actor_options,
_internal=True,
)
def generate_executor_dag_driver_deployment(
serve_executor_dag_root_node: DAGNode, original_driver_deployment: Deployment
):
"""Given a transformed minimal execution serve dag, and original DAGDriver
deployment, generate new DAGDriver deployment that uses new serve executor
dag as init_args.
Args:
serve_executor_dag_root_node (DeploymentExecutorNode): Transformed
executor serve dag with only barebone deployment handles.
original_driver_deployment (Deployment): User's original DAGDriver
deployment that wrapped Ray DAG as init args.
Returns:
executor_dag_driver_deployment (Deployment): New DAGDriver deployment
with executor serve dag as init args.
"""
def replace_with_handle(node):
if isinstance(node, DeploymentExecutorNode):
return node._deployment_handle
elif isinstance(
node,
(
DeploymentMethodExecutorNode,
DeploymentFunctionExecutorNode,
),
):
serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder)
return RayServeDAGHandle(serve_dag_root_json)
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = serve_executor_dag_root_node.apply_functional(
[
serve_executor_dag_root_node.get_args(),
serve_executor_dag_root_node.get_kwargs(),
],
predictate_fn=lambda node: isinstance(
node,
(
DeploymentExecutorNode,
DeploymentFunctionExecutorNode,
DeploymentMethodExecutorNode,
),
),
apply_fn=replace_with_handle,
)
return original_driver_deployment.options(
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
)
def from_json(cls, input_json, object_hook=None):
assert input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__
return cls(
Deployment(
input_json["import_path"],
input_json["deployment_name"],
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=input_json["args"],
init_kwargs=input_json["kwargs"],
ray_actor_options=input_json["options"],
_internal=True,
),
input_json["deployment_method_name"],
input_json["args"],
input_json["kwargs"],
input_json["options"],
other_args_to_resolve=input_json["other_args_to_resolve"],
)
def list_deployments() -> Dict[str, Deployment]:
"""Returns a dictionary of all active deployments.
Dictionary maps deployment name to Deployment objects.
"""
infos = get_global_client().list_deployments()
deployments = {}
for name, (deployment_info, route_prefix) in infos.items():
deployments[name] = Deployment(
deployment_info.replica_config.deployment_def,
name,
deployment_info.deployment_config,
version=deployment_info.version,
init_args=deployment_info.replica_config.init_args,
init_kwargs=deployment_info.replica_config.init_kwargs,
route_prefix=route_prefix,
ray_actor_options=deployment_info.replica_config.ray_actor_options,
_internal=True,
)
return deployments
def get_deployment(name: str) -> Deployment:
"""Dynamically fetch a handle to a Deployment object.
This can be used to update and redeploy a deployment without access to
the original definition.
Example:
>>> from ray import serve
>>> MyDeployment = serve.get_deployment("name") # doctest: +SKIP
>>> MyDeployment.options(num_replicas=10).deploy() # doctest: +SKIP
Args:
name(str): name of the deployment. This must have already been
deployed.
Returns:
Deployment
"""
try:
(
deployment_info,
route_prefix,
) = get_global_client().get_deployment_info(name)
except KeyError:
raise KeyError(
f"Deployment {name} was not found. Did you call Deployment.deploy()?"
)
return Deployment(
cloudpickle.loads(
deployment_info.replica_config.serialized_deployment_def),
name,
deployment_info.deployment_config,
version=deployment_info.version,
init_args=deployment_info.replica_config.init_args,
init_kwargs=deployment_info.replica_config.init_kwargs,
route_prefix=route_prefix,
ray_actor_options=deployment_info.replica_config.ray_actor_options,
_internal=True,
)
def __init__(
self,
# For serve structured deployment, deployment body can be import path
# to the class or function instead.
func_or_class: Union[Callable, str],
deployment_name: str,
deployment_init_args: Tuple[Any],
deployment_init_kwargs: Dict[str, Any],
ray_actor_options: Dict[str, Any],
other_args_to_resolve: Optional[Dict[str, Any]] = None,
):
# Assign instance variables in base class constructor.
super().__init__(
deployment_init_args,
deployment_init_kwargs,
ray_actor_options,
other_args_to_resolve=other_args_to_resolve,
)
if self._contains_input_node():
raise ValueError(
"InputNode handles user dynamic input the the DAG, and "
"cannot be used as args, kwargs, or other_args_to_resolve "
"in the DeploymentNode constructor because it is not available "
"at class construction or binding time.")
# Deployment can be passed into other DAGNodes as init args. This is
# supported pattern in ray DAG that user can instantiate and pass class
# instances as init args to others.
# However in ray serve we send init args via .remote() that requires
# pickling, and all DAGNode types are not picklable by design.
# Thus we need convert all DeploymentNode used in init args into
# deployment handles (executable and picklable) in ray serve DAG to make
# serve DAG end to end executable.
def replace_with_handle(node):
if isinstance(node, DeploymentNode):
return node._get_serve_deployment_handle(
node._deployment, node._bound_other_args_to_resolve)
elif isinstance(node,
(DeploymentMethodNode, DeploymentFunctionNode)):
from ray.serve.pipeline.json_serde import DAGNodeEncoder
serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder)
return RayServeDAGHandle(serve_dag_root_json)
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = self.apply_functional(
[deployment_init_args, deployment_init_kwargs],
predictate_fn=lambda node: isinstance(node, (
DeploymentNode, DeploymentMethodNode, DeploymentFunctionNode)),
apply_fn=replace_with_handle,
)
if "deployment_schema" in self._bound_other_args_to_resolve:
deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[
"deployment_schema"]
deployment_shell = schema_to_deployment(deployment_schema)
# Prefer user specified name to override the generated one.
if (inspect.isclass(func_or_class)
and deployment_shell.name != func_or_class.__name__):
deployment_name = deployment_shell.name
# Set the route prefix, prefer the one user supplied,
# otherwise set it to /deployment_name
if (deployment_shell.route_prefix is None
or deployment_shell.route_prefix !=
f"/{deployment_shell.name}"):
route_prefix = deployment_shell.route_prefix
else:
route_prefix = f"/{deployment_name}"
self._deployment = deployment_shell.options(
func_or_class=func_or_class,
name=deployment_name,
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
route_prefix=route_prefix,
)
else:
self._deployment: Deployment = Deployment(
func_or_class,
deployment_name,
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
ray_actor_options=ray_actor_options,
_internal=True,
)
self._deployment_handle: Union[
RayServeLazySyncHandle, RayServeHandle,
RayServeSyncHandle] = self._get_serve_deployment_handle(
self._deployment, other_args_to_resolve)
def __init__(
self,
# For serve structured deployment, deployment body can be import path
# to the class or function instead.
func_or_class: Union[Callable, str],
deployment_name: str,
deployment_init_args: Tuple[Any],
deployment_init_kwargs: Dict[str, Any],
ray_actor_options: Dict[str, Any],
other_args_to_resolve: Optional[Dict[str, Any]] = None,
):
# Assign instance variables in base class constructor.
super().__init__(
deployment_init_args,
deployment_init_kwargs,
ray_actor_options,
other_args_to_resolve=other_args_to_resolve,
)
# Deployment can be passed into other DAGNodes as init args. This is
# supported pattern in ray DAG that user can instantiate and pass class
# instances as init args to others.
# However in ray serve we send init args via .remote() that requires
# pickling, and all DAGNode types are not picklable by design.
# Thus we need convert all DeploymentNode used in init args into
# deployment handles (executable and picklable) in ray serve DAG to make
# serve DAG end to end executable.
# TODO(jiaodong): This part does some magic for DAGDriver and will throw
# error with weird pickle replace table error. Move this out.
def replace_with_handle(node):
if isinstance(node, DeploymentNode):
return RayServeLazySyncHandle(node._deployment.name)
elif isinstance(node, DeploymentExecutorNode):
return node._deployment_handle
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = self.apply_functional(
[deployment_init_args, deployment_init_kwargs],
predictate_fn=lambda node: isinstance(
node,
# We need to match and replace all DAGNodes even though they
# could be None, because no DAGNode replacement should run into
# re-resolved child DAGNodes, otherwise with KeyError
(
DeploymentNode,
DeploymentMethodNode,
DeploymentFunctionNode,
DeploymentExecutorNode,
DeploymentFunctionExecutorNode,
DeploymentMethodExecutorNode,
),
),
apply_fn=replace_with_handle,
)
if "deployment_schema" in self._bound_other_args_to_resolve:
deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[
"deployment_schema"
]
deployment_shell = schema_to_deployment(deployment_schema)
# Prefer user specified name to override the generated one.
if (
inspect.isclass(func_or_class)
and deployment_shell.name != func_or_class.__name__
):
deployment_name = deployment_shell.name
# Set the route prefix, prefer the one user supplied,
# otherwise set it to /deployment_name
if (
deployment_shell.route_prefix is None
or deployment_shell.route_prefix != f"/{deployment_shell.name}"
):
route_prefix = deployment_shell.route_prefix
else:
route_prefix = f"/{deployment_name}"
self._deployment = deployment_shell.options(
func_or_class=func_or_class,
name=deployment_name,
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
route_prefix=route_prefix,
)
else:
self._deployment: Deployment = Deployment(
func_or_class,
deployment_name,
# TODO: (jiaodong) Support deployment config from user input
DeploymentConfig(),
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
ray_actor_options=ray_actor_options,
_internal=True,
)
self._deployment_handle = RayServeLazySyncHandle(self._deployment.name)
