def _get_serve_deployment_handle( self, deployment: Deployment, bound_other_args_to_resolve: Dict[str, Any], ) -> Union[RayServeHandle, RayServeSyncHandle]: """ Return a sync or async handle of the encapsulated Deployment based on config. Args: deployment (Deployment): Deployment instance wrapped in the DAGNode. bound_other_args_to_resolve (Dict[str, Any]): Contains args used to configure DeploymentNode. Returns: RayServeHandle: Default and catch-all is to return sync handle. return async handle only if user explicitly set USE_SYNC_HANDLE_KEY with value of False. """ # TODO (jiaodong): Support configurable async handle if USE_SYNC_HANDLE_KEY not in bound_other_args_to_resolve: # Return sync RayServeLazySyncHandle return RayServeLazySyncHandle(deployment.name) elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is True: # Return sync RayServeSyncHandle return deployment.get_handle(sync=True) elif bound_other_args_to_resolve.get(USE_SYNC_HANDLE_KEY) is False: # Return async RayServeHandle return deployment.get_handle(sync=False) else: raise ValueError( f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value." )
def test_invalid_use_sync_handle(): deployment = Deployment( Actor, "test", DeploymentConfig(), _internal=True, ) with pytest.raises( ValueError, match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value", ): _ = DeploymentNode( Actor, "test", [], {}, {}, other_args_to_resolve={USE_SYNC_HANDLE_KEY: {"options_a": "hii"}}, ) with pytest.raises( ValueError, match=f"{USE_SYNC_HANDLE_KEY} should only be set with a boolean value", ): _ = DeploymentMethodNode( deployment, "method", [], {}, {}, other_args_to_resolve={ USE_SYNC_HANDLE_KEY: None, }, )
def __init__( self, func_body: Union[Callable, str], deployment_name, func_args, func_kwargs, func_options, other_args_to_resolve=None, ): self._body = func_body self._deployment_name = deployment_name super().__init__( func_args, func_kwargs, func_options, other_args_to_resolve=other_args_to_resolve, ) if "deployment_schema" in self._bound_other_args_to_resolve: deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[ "deployment_schema" ] deployment_shell = schema_to_deployment(deployment_schema) # Prefer user specified name to override the generated one. if ( inspect.isfunction(func_body) and deployment_shell.name != func_body.__name__ ): self._deployment_name = deployment_shell.name # Set the route prefix, prefer the one user supplied, # otherwise set it to /deployment_name if ( deployment_shell.route_prefix is None or deployment_shell.route_prefix != f"/{deployment_shell.name}" ): route_prefix = deployment_shell.route_prefix else: route_prefix = f"/{deployment_name}" self._deployment = deployment_shell.options( func_or_class=func_body, name=self._deployment_name, init_args=(), init_kwargs={}, route_prefix=route_prefix, ) else: self._deployment: Deployment = Deployment( func_body, deployment_name, DeploymentConfig(), init_args=tuple(), init_kwargs=dict(), ray_actor_options=func_options, _internal=True, ) # TODO (jiaodong): Polish with async handle support later self._deployment_handle = RayServeLazySyncHandle(self._deployment.name)
def decorator(_func_or_class): return Deployment( _func_or_class, name if name is not None else _func_or_class.__name__, config, version=version, init_args=init_args, init_kwargs=init_kwargs, route_prefix=route_prefix, ray_actor_options=ray_actor_options, _internal=True, )
def generate_executor_dag_driver_deployment( serve_executor_dag_root_node: DAGNode, original_driver_deployment: Deployment ): """Given a transformed minimal execution serve dag, and original DAGDriver deployment, generate new DAGDriver deployment that uses new serve executor dag as init_args. Args: serve_executor_dag_root_node (DeploymentExecutorNode): Transformed executor serve dag with only barebone deployment handles. original_driver_deployment (Deployment): User's original DAGDriver deployment that wrapped Ray DAG as init args. Returns: executor_dag_driver_deployment (Deployment): New DAGDriver deployment with executor serve dag as init args. """ def replace_with_handle(node): if isinstance(node, DeploymentExecutorNode): return node._deployment_handle elif isinstance( node, ( DeploymentMethodExecutorNode, DeploymentFunctionExecutorNode, ), ): serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder) return RayServeDAGHandle(serve_dag_root_json) ( replaced_deployment_init_args, replaced_deployment_init_kwargs, ) = serve_executor_dag_root_node.apply_functional( [ serve_executor_dag_root_node.get_args(), serve_executor_dag_root_node.get_kwargs(), ], predictate_fn=lambda node: isinstance( node, ( DeploymentExecutorNode, DeploymentFunctionExecutorNode, DeploymentMethodExecutorNode, ), ), apply_fn=replace_with_handle, ) return original_driver_deployment.options( init_args=replaced_deployment_init_args, init_kwargs=replaced_deployment_init_kwargs, )
def from_json(cls, input_json, object_hook=None): assert input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__ return cls( Deployment( input_json["import_path"], input_json["deployment_name"], # TODO: (jiaodong) Support deployment config from user input DeploymentConfig(), init_args=input_json["args"], init_kwargs=input_json["kwargs"], ray_actor_options=input_json["options"], _internal=True, ), input_json["deployment_method_name"], input_json["args"], input_json["kwargs"], input_json["options"], other_args_to_resolve=input_json["other_args_to_resolve"], )
def list_deployments() -> Dict[str, Deployment]: """Returns a dictionary of all active deployments. Dictionary maps deployment name to Deployment objects. """ infos = get_global_client().list_deployments() deployments = {} for name, (deployment_info, route_prefix) in infos.items(): deployments[name] = Deployment( deployment_info.replica_config.deployment_def, name, deployment_info.deployment_config, version=deployment_info.version, init_args=deployment_info.replica_config.init_args, init_kwargs=deployment_info.replica_config.init_kwargs, route_prefix=route_prefix, ray_actor_options=deployment_info.replica_config.ray_actor_options, _internal=True, ) return deployments
def get_deployment(name: str) -> Deployment: """Dynamically fetch a handle to a Deployment object. This can be used to update and redeploy a deployment without access to the original definition. Example: >>> from ray import serve >>> MyDeployment = serve.get_deployment("name") # doctest: +SKIP >>> MyDeployment.options(num_replicas=10).deploy() # doctest: +SKIP Args: name(str): name of the deployment. This must have already been deployed. Returns: Deployment """ try: ( deployment_info, route_prefix, ) = get_global_client().get_deployment_info(name) except KeyError: raise KeyError( f"Deployment {name} was not found. Did you call Deployment.deploy()?" ) return Deployment( cloudpickle.loads( deployment_info.replica_config.serialized_deployment_def), name, deployment_info.deployment_config, version=deployment_info.version, init_args=deployment_info.replica_config.init_args, init_kwargs=deployment_info.replica_config.init_kwargs, route_prefix=route_prefix, ray_actor_options=deployment_info.replica_config.ray_actor_options, _internal=True, )
def __init__( self, # For serve structured deployment, deployment body can be import path # to the class or function instead. func_or_class: Union[Callable, str], deployment_name: str, deployment_init_args: Tuple[Any], deployment_init_kwargs: Dict[str, Any], ray_actor_options: Dict[str, Any], other_args_to_resolve: Optional[Dict[str, Any]] = None, ): # Assign instance variables in base class constructor. super().__init__( deployment_init_args, deployment_init_kwargs, ray_actor_options, other_args_to_resolve=other_args_to_resolve, ) if self._contains_input_node(): raise ValueError( "InputNode handles user dynamic input the the DAG, and " "cannot be used as args, kwargs, or other_args_to_resolve " "in the DeploymentNode constructor because it is not available " "at class construction or binding time.") # Deployment can be passed into other DAGNodes as init args. This is # supported pattern in ray DAG that user can instantiate and pass class # instances as init args to others. # However in ray serve we send init args via .remote() that requires # pickling, and all DAGNode types are not picklable by design. # Thus we need convert all DeploymentNode used in init args into # deployment handles (executable and picklable) in ray serve DAG to make # serve DAG end to end executable. def replace_with_handle(node): if isinstance(node, DeploymentNode): return node._get_serve_deployment_handle( node._deployment, node._bound_other_args_to_resolve) elif isinstance(node, (DeploymentMethodNode, DeploymentFunctionNode)): from ray.serve.pipeline.json_serde import DAGNodeEncoder serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder) return RayServeDAGHandle(serve_dag_root_json) ( replaced_deployment_init_args, replaced_deployment_init_kwargs, ) = self.apply_functional( [deployment_init_args, deployment_init_kwargs], predictate_fn=lambda node: isinstance(node, ( DeploymentNode, DeploymentMethodNode, DeploymentFunctionNode)), apply_fn=replace_with_handle, ) if "deployment_schema" in self._bound_other_args_to_resolve: deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[ "deployment_schema"] deployment_shell = schema_to_deployment(deployment_schema) # Prefer user specified name to override the generated one. if (inspect.isclass(func_or_class) and deployment_shell.name != func_or_class.__name__): deployment_name = deployment_shell.name # Set the route prefix, prefer the one user supplied, # otherwise set it to /deployment_name if (deployment_shell.route_prefix is None or deployment_shell.route_prefix != f"/{deployment_shell.name}"): route_prefix = deployment_shell.route_prefix else: route_prefix = f"/{deployment_name}" self._deployment = deployment_shell.options( func_or_class=func_or_class, name=deployment_name, init_args=replaced_deployment_init_args, init_kwargs=replaced_deployment_init_kwargs, route_prefix=route_prefix, ) else: self._deployment: Deployment = Deployment( func_or_class, deployment_name, # TODO: (jiaodong) Support deployment config from user input DeploymentConfig(), init_args=replaced_deployment_init_args, init_kwargs=replaced_deployment_init_kwargs, ray_actor_options=ray_actor_options, _internal=True, ) self._deployment_handle: Union[ RayServeLazySyncHandle, RayServeHandle, RayServeSyncHandle] = self._get_serve_deployment_handle( self._deployment, other_args_to_resolve)
def __init__( self, # For serve structured deployment, deployment body can be import path # to the class or function instead. func_or_class: Union[Callable, str], deployment_name: str, deployment_init_args: Tuple[Any], deployment_init_kwargs: Dict[str, Any], ray_actor_options: Dict[str, Any], other_args_to_resolve: Optional[Dict[str, Any]] = None, ): # Assign instance variables in base class constructor. super().__init__( deployment_init_args, deployment_init_kwargs, ray_actor_options, other_args_to_resolve=other_args_to_resolve, ) # Deployment can be passed into other DAGNodes as init args. This is # supported pattern in ray DAG that user can instantiate and pass class # instances as init args to others. # However in ray serve we send init args via .remote() that requires # pickling, and all DAGNode types are not picklable by design. # Thus we need convert all DeploymentNode used in init args into # deployment handles (executable and picklable) in ray serve DAG to make # serve DAG end to end executable. # TODO(jiaodong): This part does some magic for DAGDriver and will throw # error with weird pickle replace table error. Move this out. def replace_with_handle(node): if isinstance(node, DeploymentNode): return RayServeLazySyncHandle(node._deployment.name) elif isinstance(node, DeploymentExecutorNode): return node._deployment_handle ( replaced_deployment_init_args, replaced_deployment_init_kwargs, ) = self.apply_functional( [deployment_init_args, deployment_init_kwargs], predictate_fn=lambda node: isinstance( node, # We need to match and replace all DAGNodes even though they # could be None, because no DAGNode replacement should run into # re-resolved child DAGNodes, otherwise with KeyError ( DeploymentNode, DeploymentMethodNode, DeploymentFunctionNode, DeploymentExecutorNode, DeploymentFunctionExecutorNode, DeploymentMethodExecutorNode, ), ), apply_fn=replace_with_handle, ) if "deployment_schema" in self._bound_other_args_to_resolve: deployment_schema: DeploymentSchema = self._bound_other_args_to_resolve[ "deployment_schema" ] deployment_shell = schema_to_deployment(deployment_schema) # Prefer user specified name to override the generated one. if ( inspect.isclass(func_or_class) and deployment_shell.name != func_or_class.__name__ ): deployment_name = deployment_shell.name # Set the route prefix, prefer the one user supplied, # otherwise set it to /deployment_name if ( deployment_shell.route_prefix is None or deployment_shell.route_prefix != f"/{deployment_shell.name}" ): route_prefix = deployment_shell.route_prefix else: route_prefix = f"/{deployment_name}" self._deployment = deployment_shell.options( func_or_class=func_or_class, name=deployment_name, init_args=replaced_deployment_init_args, init_kwargs=replaced_deployment_init_kwargs, route_prefix=route_prefix, ) else: self._deployment: Deployment = Deployment( func_or_class, deployment_name, # TODO: (jiaodong) Support deployment config from user input DeploymentConfig(), init_args=replaced_deployment_init_args, init_kwargs=replaced_deployment_init_kwargs, ray_actor_options=ray_actor_options, _internal=True, ) self._deployment_handle = RayServeLazySyncHandle(self._deployment.name)