def test_http_options(): HTTPOptions() HTTPOptions(host="8.8.8.8", middlewares=[object()]) assert HTTPOptions(host=None).location == "NoServer" assert HTTPOptions(location=None).location == "NoServer" assert HTTPOptions( location=DeploymentMode.EveryNode).location == "EveryNode"
def test_node_selection(patch_get_namespace): def _make_http_state(http_options): return HTTPState( "mock_controller_name", detached=True, config=http_options, _start_proxies_on_init=False, ) all_nodes = [("node_id-index-head", "node-id-1")] + [ (f"node_idx-worker-{i}", f"node-id-{i}") for i in range(100) ] with patch("ray.serve.http_state.get_all_node_ids") as func: func.return_value = all_nodes # Test NoServer state = _make_http_state(HTTPOptions(location=DeploymentMode.NoServer)) assert state._get_target_nodes() == [] # Test HeadOnly with patch("ray.serve.http_state.get_current_node_resource_key" ) as get_current_node: get_current_node.return_value = "node-id-1" state = _make_http_state( HTTPOptions(location=DeploymentMode.HeadOnly)) assert state._get_target_nodes() == all_nodes[:1] # Test EveryNode state = _make_http_state( HTTPOptions(location=DeploymentMode.EveryNode)) assert state._get_target_nodes() == all_nodes # Test FixedReplica state = _make_http_state( HTTPOptions(location=DeploymentMode.FixedNumber, fixed_number_replicas=5)) selected_nodes = state._get_target_nodes() # it should have selection a subset of 5 nodes. assert len(selected_nodes) == 5 assert set(all_nodes).issuperset(set(selected_nodes)) for _ in range(5): # The selection should be deterministic. assert selected_nodes == state._get_target_nodes() another_seed = _make_http_state( HTTPOptions( location=DeploymentMode.FixedNumber, fixed_number_replicas=5, fixed_number_selection_seed=42, ))._get_target_nodes() assert len(another_seed) == 5 assert set(all_nodes).issuperset(set(another_seed)) assert set(another_seed) != set(selected_nodes)
def _check_http_and_checkpoint_options( client: ServeControllerClient, http_options: Union[dict, HTTPOptions], checkpoint_path: str, ) -> None: if checkpoint_path and checkpoint_path != client.checkpoint_path: logger.warning( f"The new client checkpoint path '{checkpoint_path}' " f"is different from the existing one '{client.checkpoint_path}'. " "The new checkpoint path is ignored.") if http_options: client_http_options = client.http_config new_http_options = (http_options if isinstance(http_options, HTTPOptions) else HTTPOptions.parse_obj(http_options)) different_fields = [] all_http_option_fields = new_http_options.__dict__ for field in all_http_option_fields: if getattr(new_http_options, field) != getattr( client_http_options, field): different_fields.append(field) if len(different_fields): logger.warning( "The new client HTTP config differs from the existing one " f"in the following fields: {different_fields}. " "The new HTTP config is ignored.")
def __init__( self, controller_name: str, checkpoint_path: str, detached: bool = False, dedicated_cpu: bool = False, http_proxy_port: int = 8000, ): try: self._controller = ray.get_actor(controller_name, namespace="serve") except ValueError: self._controller = None if self._controller is None: # Used for scheduling things to the head node explicitly. head_node_id = ray.get_runtime_context().node_id.hex() http_config = HTTPOptions() http_config.port = http_proxy_port self._controller = ServeController.options( num_cpus=1 if dedicated_cpu else 0, name=controller_name, lifetime="detached" if detached else None, max_restarts=-1, max_task_retries=-1, # Schedule the controller on the head node with a soft constraint. This # prefers it to run on the head node in most cases, but allows it to be # restarted on other nodes in an HA cluster. scheduling_strategy=NodeAffinitySchedulingStrategy( head_node_id, soft=True ), namespace="serve", max_concurrency=CONTROLLER_MAX_CONCURRENCY, ).remote( controller_name, http_config=http_config, checkpoint_path=checkpoint_path, head_node_id=head_node_id, detached=detached, )
def test_dedicated_cpu(controller_cpu, num_proxy_cpus, ray_cluster): cluster = ray_cluster num_cluster_cpus = 8 head_node = cluster.add_node(num_cpus=num_cluster_cpus) ray.init(head_node.address) wait_for_condition(lambda: ray.cluster_resources().get("CPU") == num_cluster_cpus) num_cpus_used = int(controller_cpu) + num_proxy_cpus serve.start( dedicated_cpu=controller_cpu, http_options=HTTPOptions(num_cpus=num_proxy_cpus) ) available_cpus = num_cluster_cpus - num_cpus_used wait_for_condition(lambda: (ray.available_resources().get("CPU") == available_cpus)) serve.shutdown() ray.shutdown()
def __init__( self, controller_name: str, detached: bool, config: HTTPOptions, head_node_id: str, # Used by unit testing _start_proxies_on_init: bool = True, ): self._controller_name = controller_name self._detached = detached if config is not None: self._config = config else: self._config = HTTPOptions() self._proxy_actors: Dict[NodeId, ActorHandle] = dict() self._proxy_actor_names: Dict[NodeId, str] = dict() self._head_node_id: str = head_node_id assert isinstance(head_node_id, str) # Will populate self.proxy_actors with existing actors. if _start_proxies_on_init: self._start_proxies_if_needed()
def start( detached: bool = False, http_options: Optional[Union[dict, HTTPOptions]] = None, dedicated_cpu: bool = False, _checkpoint_path: str = DEFAULT_CHECKPOINT_PATH, **kwargs, ) -> ServeControllerClient: """Initialize a serve instance. By default, the instance will be scoped to the lifetime of the returned Client object (or when the script exits). If detached is set to True, the instance will instead persist until serve.shutdown() is called. This is only relevant if connecting to a long-running Ray cluster (e.g., with ray.init(address="auto") or ray.init("ray://<remote_addr>")). Args: detached: Whether not the instance should be detached from this script. If set, the instance will live on the Ray cluster until it is explicitly stopped with serve.shutdown(). http_options (Optional[Dict, serve.HTTPOptions]): Configuration options for HTTP proxy. You can pass in a dictionary or HTTPOptions object with fields: - host(str, None): Host for HTTP servers to listen on. Defaults to "127.0.0.1". To expose Serve publicly, you probably want to set this to "0.0.0.0". - port(int): Port for HTTP server. Defaults to 8000. - root_path(str): Root path to mount the serve application (for example, "/serve"). All deployment routes will be prefixed with this path. Defaults to "". - middlewares(list): A list of Starlette middlewares that will be applied to the HTTP servers in the cluster. Defaults to []. - location(str, serve.config.DeploymentMode): The deployment location of HTTP servers: - "HeadOnly": start one HTTP server on the head node. Serve assumes the head node is the node you executed serve.start on. This is the default. - "EveryNode": start one HTTP server per node. - "NoServer" or None: disable HTTP server. - num_cpus (int): The number of CPU cores to reserve for each internal Serve HTTP proxy actor. Defaults to 0. dedicated_cpu: Whether to reserve a CPU core for the internal Serve controller actor. Defaults to False. """ usage_lib.record_library_usage("serve") http_deprecated_args = ["http_host", "http_port", "http_middlewares"] for key in http_deprecated_args: if key in kwargs: raise ValueError( f"{key} is deprecated, please use serve.start(http_options=" f'{{"{key}": {kwargs[key]}}}) instead.') # Initialize ray if needed. ray._private.worker.global_worker.filter_logs_by_job = False if not ray.is_initialized(): ray.init(namespace=SERVE_NAMESPACE) try: client = get_global_client(_health_check_controller=True) logger.info( f'Connecting to existing Serve app in namespace "{SERVE_NAMESPACE}".' ) _check_http_and_checkpoint_options(client, http_options, _checkpoint_path) return client except RayServeException: pass if detached: controller_name = SERVE_CONTROLLER_NAME else: controller_name = format_actor_name(get_random_letters(), SERVE_CONTROLLER_NAME) if isinstance(http_options, dict): http_options = HTTPOptions.parse_obj(http_options) if http_options is None: http_options = HTTPOptions() controller = ServeController.options( num_cpus=1 if dedicated_cpu else 0, name=controller_name, lifetime="detached" if detached else None, max_restarts=-1, max_task_retries=-1, # Pin Serve controller on the head node. resources={ get_current_node_resource_key(): 0.01 }, namespace=SERVE_NAMESPACE, max_concurrency=CONTROLLER_MAX_CONCURRENCY, ).remote( controller_name, http_options, _checkpoint_path, detached=detached, ) proxy_handles = ray.get(controller.get_http_proxies.remote()) if len(proxy_handles) > 0: try: ray.get( [handle.ready.remote() for handle in proxy_handles.values()], timeout=HTTP_PROXY_TIMEOUT, ) except ray.exceptions.GetTimeoutError: raise TimeoutError( f"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.") client = ServeControllerClient( controller, controller_name, detached=detached, ) set_global_client(client) logger.info(f"Started{' detached ' if detached else ' '}Serve instance in " f'namespace "{SERVE_NAMESPACE}".') return client
def start( detached: bool = False, http_host: Optional[str] = DEFAULT_HTTP_HOST, http_port: int = DEFAULT_HTTP_PORT, http_middlewares: List[Any] = [], http_options: Optional[Union[dict, HTTPOptions]] = None, ) -> Client: """Initialize a serve instance. By default, the instance will be scoped to the lifetime of the returned Client object (or when the script exits). If detached is set to True, the instance will instead persist until client.shutdown() is called and clients to it can be connected using serve.connect(). This is only relevant if connecting to a long-running Ray cluster (e.g., with address="auto"). Args: detached (bool): Whether not the instance should be detached from this script. http_host (Optional[str]): Deprecated, use http_options instead. http_port (int): Deprecated, use http_options instead. http_middlewares (list): Deprecated, use http_options instead. http_options (Optional[Dict, serve.HTTPOptions]): Configuration options for HTTP proxy. You can pass in a dictionary or HTTPOptions object with fields: - host(str, None): Host for HTTP servers to listen on. Defaults to "127.0.0.1". To expose Serve publicly, you probably want to set this to "0.0.0.0". - port(int): Port for HTTP server. Defaults to 8000. - middlewares(list): A list of Starlette middlewares that will be applied to the HTTP servers in the cluster. - location(str, serve.config.DeploymentMode): The deployment location of HTTP servers: - "HeadOnly": start one HTTP server on the head node. Serve assumes the head node is the node you executed serve.start on. This is the default. - "EveryNode": start one HTTP server per node. - "NoServer" or None: disable HTTP server. """ if ((http_host != DEFAULT_HTTP_HOST) or (http_port != DEFAULT_HTTP_PORT) or (len(http_middlewares) != 0)): if http_options is not None: raise ValueError( "You cannot specify both `http_options` and any of the " "`http_host`, `http_port`, and `http_middlewares` arguments. " "`http_options` is preferred.") else: warn( "`http_host`, `http_port`, `http_middlewares` are deprecated. " "Please use serve.start(http_options={'host': ..., " "'port': ..., middlewares': ...}) instead.", DeprecationWarning, ) # Initialize ray if needed. if not ray.is_initialized(): ray.init() register_custom_serializers() # Try to get serve controller if it exists if detached: controller_name = SERVE_CONTROLLER_NAME try: ray.get_actor(controller_name) raise RayServeException("Called serve.start(detached=True) but a " "detached instance is already running. " "Please use serve.connect() to connect to " "the running instance instead.") except ValueError: pass else: controller_name = format_actor_name(SERVE_CONTROLLER_NAME, get_random_letters()) if isinstance(http_options, dict): http_options = HTTPOptions.parse_obj(http_options) if http_options is None: http_options = HTTPOptions( host=http_host, port=http_port, middlewares=http_middlewares) controller = ServeController.options( name=controller_name, lifetime="detached" if detached else None, max_restarts=-1, max_task_retries=-1, # Pin Serve controller on the head node. resources={ get_current_node_resource_key(): 0.01 }, ).remote( controller_name, http_options, detached=detached, ) proxy_handles = ray.get(controller.get_http_proxies.remote()) if len(proxy_handles) > 0: try: ray.get( [handle.ready.remote() for handle in proxy_handles.values()], timeout=HTTP_PROXY_TIMEOUT, ) except ray.exceptions.GetTimeoutError: raise TimeoutError( "HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.") client = Client(controller, controller_name, detached=detached) _set_global_client(client) return client