async def test_changing_backend(ray_instance, mock_controller, task_runner_mock_actor): q = ray.remote(EndpointRouter).remote(mock_controller, "svc") await mock_controller.set_traffic.remote( "svc", TrafficPolicy({ "backend-alter": 1 })) await mock_controller.add_new_replica.remote("backend-alter", task_runner_mock_actor) await (await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc"), 1)) got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 1 await mock_controller.set_traffic.remote( "svc", TrafficPolicy({ "backend-alter-2": 1 })) await mock_controller.add_new_replica.remote("backend-alter-2", task_runner_mock_actor) await (await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc"), 2)) got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 2
def _start_replica(backend_tag): assert (backend_tag in global_state.backend_table.list_backends() ), "Backend {} is not registered.".format(backend_tag) replica_tag = "{}#{}".format(backend_tag, get_random_letters(length=6)) # get the info which starts the replicas creator = global_state.backend_table.get_backend_creator(backend_tag) backend_config_dict = global_state.backend_table.get_info(backend_tag) backend_config = BackendConfig(**backend_config_dict) init_args = global_state.backend_table.get_init_args(backend_tag) # get actor creation kwargs actor_kwargs = backend_config.get_actor_creation_args(init_args) # Create the runner in the nursery [runner_handle] = ray.get( global_state.actor_nursery_handle.start_actor_with_creator.remote( creator, actor_kwargs, replica_tag)) # Setup the worker ray.get( runner_handle._ray_serve_setup.remote( backend_tag, global_state.init_or_get_router(), runner_handle)) runner_handle._ray_serve_fetch.remote() # Register the worker in config tables as well as metric monitor global_state.backend_table.add_replica(backend_tag, replica_tag) global_state.init_or_get_metric_monitor().add_target.remote(runner_handle)
def _scale_replicas(self, backends: Dict[BackendTag, BackendInfo], backend_tag: BackendTag, num_replicas: int) -> None: """Scale the given backend to the number of replicas. NOTE: this does not actually start or stop the replicas, but instead adds the intention to start/stop them to self.workers_to_start and self.workers_to_stop. The caller is responsible for then first writing a checkpoint and then actually starting/stopping the intended replicas. This avoids inconsistencies with starting/stopping a worker and then crashing before writing a checkpoint. """ logger.debug("Scaling backend '{}' to {} replicas".format( backend_tag, num_replicas)) assert ( backend_tag in backends), "Backend {} is not registered.".format(backend_tag) assert num_replicas >= 0, ("Number of replicas must be" " greater than or equal to 0.") current_num_replicas = len(self.replicas[backend_tag]) delta_num_replicas = num_replicas - current_num_replicas backend_info = backends[backend_tag] if delta_num_replicas > 0: can_schedule = try_schedule_resources_on_nodes(requirements=[ backend_info.replica_config.resource_dict for _ in range(delta_num_replicas) ]) if _RESOURCE_CHECK_ENABLED and not all(can_schedule): num_possible = sum(can_schedule) raise RayServeException( "Cannot scale backend {} to {} replicas. Ray Serve tried " "to add {} replicas but the resources only allows {} " "to be added. To fix this, consider scaling to replica to " "{} or add more resources to the cluster. You can check " "avaiable resources with ray.nodes().".format( backend_tag, num_replicas, delta_num_replicas, num_possible, current_num_replicas + num_possible)) logger.debug("Adding {} replicas to backend {}".format( delta_num_replicas, backend_tag)) for _ in range(delta_num_replicas): replica_tag = "{}#{}".format(backend_tag, get_random_letters()) self.replicas_to_start[backend_tag].append(replica_tag) elif delta_num_replicas < 0: logger.debug("Removing {} replicas from backend '{}'".format( -delta_num_replicas, backend_tag)) assert len(self.replicas[backend_tag]) >= delta_num_replicas for _ in range(-delta_num_replicas): replica_tag = self.replicas[backend_tag].pop() if len(self.replicas[backend_tag]) == 0: del self.replicas[backend_tag] del self.workers[backend_tag][replica_tag] if len(self.workers[backend_tag]) == 0: del self.workers[backend_tag] self.replicas_to_stop[backend_tag].append(replica_tag)
def test_replica_name_from_str(): replica_suffix = get_random_letters() actor_name = f"{ReplicaName.prefix}DeploymentA#{replica_suffix}" replica_name = ReplicaName.from_str(actor_name) assert (str(replica_name) == replica_name.replica_tag == actor_name.replace(ReplicaName.prefix, ""))
def _set_backend_goal(self, backend_tag: BackendTag, backend_info: Optional[BackendInfo]) -> None: """ Set desirable state for a given backend, identified by tag. Args: backend_tag (BackendTag): Identifier of a backend backend_info (Optional[BackendInfo]): Contains backend and replica config, if passed in as None, we're marking target backend as shutting down. """ existing_goal_id = self._backend_goals.get(backend_tag) new_goal_id = self._goal_manager.create_goal() if backend_info is not None: self._backend_metadata[backend_tag] = backend_info self._target_replicas[ backend_tag] = backend_info.backend_config.num_replicas if backend_info.version is not None: code_version = backend_info.version else: code_version = get_random_letters() self._target_versions[backend_tag] = BackendVersion( code_version, user_config=backend_info.backend_config.user_config) else: self._target_replicas[backend_tag] = 0 self._backend_goals[backend_tag] = new_goal_id return new_goal_id, existing_goal_id
def test_replica_tag_formatting(): deployment_tag = "DeploymentA" replica_suffix = get_random_letters() replica_name = ReplicaName(deployment_tag, replica_suffix) assert replica_name.replica_tag == f"{deployment_tag}#{replica_suffix}" assert str(replica_name) == f"{deployment_tag}#{replica_suffix}"
async def test_split_traffic_random(ray_instance, mock_controller, task_runner_mock_actor): q = ray.remote(Router).remote(mock_controller) await q.setup_in_async_loop.remote() await mock_controller.set_traffic.remote( "svc", TrafficPolicy({ "backend-split": 0.5, "backend-split-2": 0.5 })) runner_1, runner_2 = [mock_task_runner() for _ in range(2)] await mock_controller.add_new_replica.remote("backend-split", runner_1) await mock_controller.add_new_replica.remote("backend-split-2", runner_2) # assume 50% split, the probability of all 20 requests goes to a # single queue is 0.5^20 ~ 1-6 for _ in range(20): await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc", None), 1) got_work = [ await runner.get_recent_call.remote() for runner in (runner_1, runner_2) ] assert [g.args[0] for g in got_work] == [1, 1]
async def _start_backend_replica(self, backend_tag): assert (backend_tag in self.backend_table.list_backends() ), "Backend {} is not registered.".format(backend_tag) replica_tag = "{}#{}".format(backend_tag, get_random_letters(length=6)) # Register the worker in the DB. # TODO(edoakes): we should guarantee that if calls to the master # succeed, the cluster state has changed and if they fail, it hasn't. # Once we have master actor fault tolerance, this breaks that guarantee # because this method could fail after writing the replica to the DB. self.backend_table.add_replica(backend_tag, replica_tag) # Fetch the info to start the replica from the backend table. backend_actor = ray.remote( self.backend_table.get_backend_creator(backend_tag)) backend_config_dict = self.backend_table.get_info(backend_tag) backend_config = BackendConfig(**backend_config_dict) init_args = [ backend_tag, replica_tag, self.backend_table.get_init_args(backend_tag) ] kwargs = backend_config.get_actor_creation_args(init_args) # Start the worker. worker_handle = backend_actor._remote(**kwargs) self.tag_to_actor_handles[replica_tag] = worker_handle # Wait for the worker to start up. await worker_handle.ready.remote() await self.get_router()[0].add_new_worker.remote( backend_tag, worker_handle) # Register the worker with the metric monitor. self.get_metric_monitor()[0].add_target.remote(worker_handle)
def test_replica_name_from_str(): replica_suffix = get_random_letters() replica_tag = f"DeploymentA#{replica_suffix}" replica_name = ReplicaName.from_str(replica_tag) assert replica_name.replica_tag == replica_tag assert str(replica_tag) == replica_tag
def remote(self, request_data: Optional[Union[Dict, Any]] = None, **kwargs): """Issue an asynchrounous request to the endpoint. Returns a Ray ObjectRef whose results can be waited for or retrieved using ray.wait or ray.get, respectively. Returns: ray.ObjectRef Args: request_data(dict, Any): If it's a dictionary, the data will be available in ``request.json()`` or ``request.form()``. Otherwise, it will be available in ``request.data``. ``**kwargs``: All keyword arguments will be available in ``request.args``. """ request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. self.endpoint_name, TaskContext.Python, call_method=self.method_name or "__call__", shard_key=self.shard_key, http_method=self.http_method or "GET", http_headers=self.http_headers or dict(), ) return self.router_handle.enqueue_request.remote( request_metadata, request_data, **kwargs)
def test_is_replica_name(): replica_suffix = get_random_letters() assert not ReplicaName.is_replica_name(f"DeploymentA##{replica_suffix}") assert not ReplicaName.is_replica_name(f"DeploymentA#{replica_suffix}") assert ReplicaName.is_replica_name( f"{ReplicaName.prefix}DeploymentA#{replica_suffix}")
async def _start_backend_replica(self, backend_tag): assert (backend_tag in self.backend_table.list_backends() ), "Backend {} is not registered.".format(backend_tag) replica_tag = "{}#{}".format(backend_tag, get_random_letters(length=6)) # Fetch the info to start the replica from the backend table. creator = self.backend_table.get_backend_creator(backend_tag) backend_config_dict = self.backend_table.get_info(backend_tag) backend_config = BackendConfig(**backend_config_dict) init_args = self.backend_table.get_init_args(backend_tag) kwargs = backend_config.get_actor_creation_args(init_args) runner_handle = creator(kwargs) self.tag_to_actor_handles[replica_tag] = runner_handle # Set up the worker. await runner_handle._ray_serve_setup.remote(backend_tag, self.get_router()[0], runner_handle) ray.get(runner_handle._ray_serve_fetch.remote()) # Register the worker in config tables and metric monitor. self.backend_table.add_replica(backend_tag, replica_tag) self.get_metric_monitor()[0].add_target.remote(runner_handle)
async def __call__(self, scope, receive, send): """Implements the ASGI protocol. See details at: https://asgi.readthedocs.io/en/latest/specs/index.html. """ error_sender = self._make_error_sender(scope, receive, send) assert self.route_table is not None, ( "Route table must be set via set_route_table.") assert scope["type"] == "http" current_path = scope["path"] self.request_counter.record(1, tags={"route": current_path}) if current_path.startswith("/-/"): await self._handle_system_request(scope, receive, send) return try: endpoint_name, methods_allowed = self.route_table[current_path] except KeyError: error_message = ( "Path {} not found. " "Please ping http://.../-/routes for routing table" ).format(current_path) await error_sender(error_message, 404) return if scope["method"] not in methods_allowed: error_message = ("Methods {} not allowed. " "Available HTTP methods are {}.").format( scope["method"], methods_allowed) await error_sender(error_message, 405) return http_body_bytes = await self.receive_http_body(scope, receive, send) headers = {k.decode(): v.decode() for k, v in scope["headers"]} request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. endpoint_name, TaskContext.Web, http_method=scope["method"].upper(), call_method=headers.get("X-SERVE-CALL-METHOD".lower(), "__call__"), shard_key=headers.get("X-SERVE-SHARD-KEY".lower(), None), ) ref = await self.router.assign_request(request_metadata, scope, http_body_bytes) result = await ref if isinstance(result, RayTaskError): error_message = "Task Error. Traceback: {}.".format(result) await error_sender(error_message, 500) else: await Response(result).send(scope, receive, send)
async def test_shard_key(ray_instance, mock_controller, task_runner_mock_actor): q = ray.remote(Router).remote(mock_controller) await q.setup_in_async_loop.remote() num_backends = 5 traffic_dict = {} runners = [mock_task_runner() for _ in range(num_backends)] for i, runner in enumerate(runners): backend_name = "backend-split-" + str(i) traffic_dict[backend_name] = 1.0 / num_backends await mock_controller.add_new_replica.remote(backend_name, runner) await mock_controller.set_traffic.remote("svc", TrafficPolicy(traffic_dict)) # Generate random shard keys and send one request for each. shard_keys = [get_random_letters() for _ in range(100)] for shard_key in shard_keys: await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc", None, shard_key=shard_key), shard_key) # Log the shard keys that were assigned to each backend. runner_shard_keys = defaultdict(set) for i, runner in enumerate(runners): calls = await runner.get_all_calls.remote() for call in calls: runner_shard_keys[i].add(call.args[0]) await runner.clear_calls.remote() # Send queries with the same shard keys a second time. for shard_key in shard_keys: await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc", None, shard_key=shard_key), shard_key) # Check that the requests were all mapped to the same backends. for i, runner in enumerate(runners): calls = await runner.get_all_calls.remote() for call in calls: assert call.args[0] in runner_shard_keys[i]
def _remote(self, deployment_name, handle_options, args, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. deployment_name, call_method=handle_options.method_name, http_arg_is_pickled=self._pickled_http_request, ) coro = self.router.assign_request(request_metadata, *args, **kwargs) return coro
async def test_alter_backend(serve_instance, task_runner_mock_actor): q = ray.remote(Router).remote() await q.setup.remote("", serve_instance._controller_name) await q.set_traffic.remote("svc", TrafficPolicy({"backend-alter": 1})) await q.add_new_worker.remote("backend-alter", "replica-1", task_runner_mock_actor) await q.enqueue_request.remote( RequestMetadata(get_random_letters(10), "svc", None), 1) got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 1 await q.set_traffic.remote("svc", TrafficPolicy({"backend-alter-2": 1})) await q.add_new_worker.remote("backend-alter-2", "replica-1", task_runner_mock_actor) await q.enqueue_request.remote( RequestMetadata(get_random_letters(10), "svc", None), 2) got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 2
def test_invalid_name_from_str(): replica_suffix = get_random_letters() replica_tag = f"DeploymentA##{replica_suffix}" with pytest.raises(AssertionError): ReplicaName.from_str(replica_tag) # No prefix replica_tag = f"DeploymentA#{replica_suffix}" with pytest.raises(AssertionError): ReplicaName.from_str(replica_tag)
def _remote(self, endpoint_name, handle_options, request_data, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. endpoint_name, call_method=handle_options.method_name, shard_key=handle_options.shard_key, http_method=handle_options.http_method, http_headers=handle_options.http_headers, ) coro = self.router.assign_request(request_metadata, request_data, **kwargs) return coro
def _remote(self, request_data, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. self.endpoint_name, TaskContext.Python, call_method=self.method_name or "__call__", shard_key=self.shard_key, http_method=self.http_method or "GET", http_headers=self.http_headers or dict(), ) coro = self.router.assign_request(request_metadata, request_data, **kwargs) return coro
def __init__(self, code_version: Optional[str], user_config: Optional[Any] = None): if code_version is not None and not isinstance(code_version, str): raise TypeError( f"code_version must be str, got {type(code_version)}.") if code_version is None: self.unversioned = True self.code_version = get_random_letters() else: self.unversioned = False self.code_version = code_version self.user_config_hash = self._hash_user_config(user_config) self._hash = hash((self.code_version, self.user_config_hash))
def __init__(self, code_version: Optional[str], user_config: Optional[Any] = None): if code_version is not None and not isinstance(code_version, str): raise TypeError(f"code_version must be str, got {type(code_version)}.") if code_version is None: self.unversioned = True self.code_version = get_random_letters() else: self.unversioned = False self.code_version = code_version self.user_config = user_config # TODO(simon): make this xlang compatible pickled_user_config = pickle.dumps(user_config) self.user_config_hash = crc32(pickled_user_config) self._hash = crc32(pickled_user_config + self.code_version.encode("utf-8"))
async def test_single_prod_cons_queue(serve_instance, task_runner_mock_actor): q = ray.remote(Router).remote() await q.setup.remote("", serve_instance._controller_name) q.set_traffic.remote("svc", TrafficPolicy({"backend-single-prod": 1.0})) q.add_new_worker.remote("backend-single-prod", "replica-1", task_runner_mock_actor) # Make sure we get the request result back result = await q.enqueue_request.remote( RequestMetadata(get_random_letters(10), "svc", None), 1) assert result == "DONE" # Make sure it's the right request got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 1 assert got_work.kwargs == {}
async def test_simple_endpoint_backend_pair(ray_instance, mock_controller, task_runner_mock_actor): q = ray.remote(Router).remote(mock_controller, "svc") # Propogate configs await mock_controller.set_traffic.remote( "svc", TrafficPolicy({"backend-single-prod": 1.0})) await mock_controller.add_new_replica.remote("backend-single-prod", task_runner_mock_actor) # Make sure we get the request result back ref = await q.assign_request.remote( RequestMetadata(get_random_letters(10), "svc"), 1) result = await ref assert result == "DONE" # Make sure it's the right request got_work = await task_runner_mock_actor.get_recent_call.remote() assert got_work.args[0] == 1 assert got_work.kwargs == {}
def _set_backend_goal(self, backend_tag: BackendTag, backend_info: Optional[BackendInfo]) -> None: existing_goal_id = self._backend_goals.get(backend_tag) new_goal_id = self._goal_manager.create_goal() if backend_info is not None: self._backend_metadata[backend_tag] = backend_info self._target_replicas[ backend_tag] = backend_info.backend_config.num_replicas if backend_info.version is not None: version = backend_info.version else: version = get_random_letters() self._target_versions[backend_tag] = version else: self._target_replicas[backend_tag] = 0 self._backend_goals[backend_tag] = new_goal_id return new_goal_id, existing_goal_id
def _scale_replicas(self, backend_tag, num_replicas): """Scale the given backend to the number of replicas. NOTE: this does not actually start or stop the replicas, but instead adds the intention to start/stop them to self.workers_to_start and self.workers_to_stop. The caller is responsible for then first writing a checkpoint and then actually starting/stopping the intended replicas. This avoids inconsistencies with starting/stopping a worker and then crashing before writing a checkpoint. """ logger.debug("Scaling backend '{}' to {} replicas".format( backend_tag, num_replicas)) assert (backend_tag in self.backends ), "Backend {} is not registered.".format(backend_tag) assert num_replicas >= 0, ("Number of replicas must be" " greater than or equal to 0.") current_num_replicas = len(self.replicas[backend_tag]) delta_num_replicas = num_replicas - current_num_replicas if delta_num_replicas > 0: logger.debug("Adding {} replicas to backend {}".format( delta_num_replicas, backend_tag)) for _ in range(delta_num_replicas): replica_tag = "{}#{}".format(backend_tag, get_random_letters()) self.replicas_to_start[backend_tag].append(replica_tag) elif delta_num_replicas < 0: logger.debug("Removing {} replicas from backend {}".format( -delta_num_replicas, backend_tag)) assert len(self.replicas[backend_tag]) >= delta_num_replicas for _ in range(-delta_num_replicas): replica_tag = self.replicas[backend_tag].pop() if len(self.replicas[backend_tag]) == 0: del self.replicas[backend_tag] del self.workers[backend_tag][replica_tag] if len(self.workers[backend_tag]) == 0: del self.workers[backend_tag] self.replicas_to_stop[backend_tag].append(replica_tag)
def test_shard_key(serve_instance, route): client = serve_instance # Create five backends that return different integers. num_backends = 5 traffic_dict = {} for i in range(num_backends): def function(_): return i backend_name = "backend-split-" + str(i) traffic_dict[backend_name] = 1.0 / num_backends client.create_backend(backend_name, function) client.create_endpoint("endpoint", backend=list(traffic_dict.keys())[0], route=route) client.set_traffic("endpoint", traffic_dict) def do_request(shard_key): if route is not None: url = "http://127.0.0.1:8000" + route headers = {"X-SERVE-SHARD-KEY": shard_key} result = requests.get(url, headers=headers).text else: handle = client.get_handle("endpoint").options(shard_key=shard_key) result = ray.get(handle.options(shard_key=shard_key).remote()) return result # Send requests with different shard keys and log the backends they go to. shard_keys = [get_random_letters() for _ in range(20)] results = {} for shard_key in shard_keys: results[shard_key] = do_request(shard_key) # Check that the shard keys are mapped to the same backends. for shard_key in shard_keys: assert do_request(shard_key) == results[shard_key]
def start( detached: bool = False, http_options: Optional[Union[dict, HTTPOptions]] = None, dedicated_cpu: bool = False, _checkpoint_path: str = DEFAULT_CHECKPOINT_PATH, **kwargs, ) -> ServeControllerClient: """Initialize a serve instance. By default, the instance will be scoped to the lifetime of the returned Client object (or when the script exits). If detached is set to True, the instance will instead persist until serve.shutdown() is called. This is only relevant if connecting to a long-running Ray cluster (e.g., with ray.init(address="auto") or ray.init("ray://<remote_addr>")). Args: detached: Whether not the instance should be detached from this script. If set, the instance will live on the Ray cluster until it is explicitly stopped with serve.shutdown(). http_options (Optional[Dict, serve.HTTPOptions]): Configuration options for HTTP proxy. You can pass in a dictionary or HTTPOptions object with fields: - host(str, None): Host for HTTP servers to listen on. Defaults to "127.0.0.1". To expose Serve publicly, you probably want to set this to "0.0.0.0". - port(int): Port for HTTP server. Defaults to 8000. - root_path(str): Root path to mount the serve application (for example, "/serve"). All deployment routes will be prefixed with this path. Defaults to "". - middlewares(list): A list of Starlette middlewares that will be applied to the HTTP servers in the cluster. Defaults to []. - location(str, serve.config.DeploymentMode): The deployment location of HTTP servers: - "HeadOnly": start one HTTP server on the head node. Serve assumes the head node is the node you executed serve.start on. This is the default. - "EveryNode": start one HTTP server per node. - "NoServer" or None: disable HTTP server. - num_cpus (int): The number of CPU cores to reserve for each internal Serve HTTP proxy actor. Defaults to 0. dedicated_cpu: Whether to reserve a CPU core for the internal Serve controller actor. Defaults to False. """ usage_lib.record_library_usage("serve") http_deprecated_args = ["http_host", "http_port", "http_middlewares"] for key in http_deprecated_args: if key in kwargs: raise ValueError( f"{key} is deprecated, please use serve.start(http_options=" f'{{"{key}": {kwargs[key]}}}) instead.') # Initialize ray if needed. ray._private.worker.global_worker.filter_logs_by_job = False if not ray.is_initialized(): ray.init(namespace=SERVE_NAMESPACE) try: client = get_global_client(_health_check_controller=True) logger.info( f'Connecting to existing Serve app in namespace "{SERVE_NAMESPACE}".' ) _check_http_and_checkpoint_options(client, http_options, _checkpoint_path) return client except RayServeException: pass if detached: controller_name = SERVE_CONTROLLER_NAME else: controller_name = format_actor_name(get_random_letters(), SERVE_CONTROLLER_NAME) if isinstance(http_options, dict): http_options = HTTPOptions.parse_obj(http_options) if http_options is None: http_options = HTTPOptions() controller = ServeController.options( num_cpus=1 if dedicated_cpu else 0, name=controller_name, lifetime="detached" if detached else None, max_restarts=-1, max_task_retries=-1, # Pin Serve controller on the head node. resources={ get_current_node_resource_key(): 0.01 }, namespace=SERVE_NAMESPACE, max_concurrency=CONTROLLER_MAX_CONCURRENCY, ).remote( controller_name, http_options, _checkpoint_path, detached=detached, ) proxy_handles = ray.get(controller.get_http_proxies.remote()) if len(proxy_handles) > 0: try: ray.get( [handle.ready.remote() for handle in proxy_handles.values()], timeout=HTTP_PROXY_TIMEOUT, ) except ray.exceptions.GetTimeoutError: raise TimeoutError( f"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.") client = ServeControllerClient( controller, controller_name, detached=detached, ) set_global_client(client) logger.info(f"Started{' detached ' if detached else ' '}Serve instance in " f'namespace "{SERVE_NAMESPACE}".') return client
def start(detached: bool = False, http_host: str = DEFAULT_HTTP_HOST, http_port: int = DEFAULT_HTTP_PORT, http_middlewares: List[Any] = []) -> Client: """Initialize a serve instance. By default, the instance will be scoped to the lifetime of the returned Client object (or when the script exits). If detached is set to True, the instance will instead persist until client.shutdown() is called and clients to it can be connected using serve.connect(). This is only relevant if connecting to a long-running Ray cluster (e.g., with address="auto"). Args: detached (bool): Whether not the instance should be detached from this script. http_host (str): Host for HTTP servers to listen on. Defaults to "127.0.0.1". To expose Serve publicly, you probably want to set this to "0.0.0.0". One HTTP server will be started on each node in the Ray cluster. http_port (int): Port for HTTP server. Defaults to 8000. http_middlewares (list): A list of Starlette middlewares that will be applied to the HTTP servers in the cluster. """ # Initialize ray if needed. if not ray.is_initialized(): ray.init() # Try to get serve controller if it exists if detached: controller_name = SERVE_CONTROLLER_NAME try: ray.get_actor(controller_name) raise RayServeException("Called serve.start(detached=True) but a " "detached instance is already running. " "Please use serve.connect() to connect to " "the running instance instead.") except ValueError: pass else: controller_name = format_actor_name(SERVE_CONTROLLER_NAME, get_random_letters()) controller = ServeController.options( name=controller_name, lifetime="detached" if detached else None, max_restarts=-1, max_task_retries=-1, ).remote(controller_name, http_host, http_port, http_middlewares, detached=detached) futures = [] for node_id in ray.state.node_ids(): future = block_until_http_ready.options( num_cpus=0, resources={ node_id: 0.01 }).remote("http://{}:{}/-/routes".format(http_host, http_port), timeout=HTTP_PROXY_TIMEOUT) futures.append(future) ray.get(futures) return Client(controller, controller_name, detached=detached)
def _scale_backend_replicas( self, backend_tag: BackendTag, target_replicas: int, target_version: str, ) -> bool: """Scale the given backend to the number of replicas. NOTE: this does not actually start or stop the replicas, but instead adds them to ReplicaState.SHOULD_START or ReplicaState.SHOULD_STOP. The caller is responsible for then first writing a checkpoint and then actually starting/stopping the intended replicas. This avoids inconsistencies with starting/stopping a replica and then crashing before writing a checkpoint. """ assert (backend_tag in self._backend_metadata ), "Backend {} is not registered.".format(backend_tag) assert target_replicas >= 0, ("Number of replicas must be" " greater than or equal to 0.") backend_info: BackendInfo = self._backend_metadata[backend_tag] graceful_shutdown_timeout_s = ( backend_info.backend_config. experimental_graceful_shutdown_timeout_s) stopped = self._stop_wrong_version_replicas( self._replicas[backend_tag], target_replicas, target_version, graceful_shutdown_timeout_s) if stopped > 0: logger.info(f"Stopping {stopped} replicas of backend " f"'{backend_tag}' with outdated versions.") current_replicas = self._replicas[backend_tag].count(states=[ ReplicaState.SHOULD_START, ReplicaState.STARTING, ReplicaState.RUNNING ]) delta_replicas = target_replicas - current_replicas if delta_replicas == 0: return False elif delta_replicas > 0: # Don't ever exceed target_replicas. stopping_replicas = self._replicas[backend_tag].count(states=[ ReplicaState.SHOULD_STOP, ReplicaState.STOPPING, ]) to_add = max(delta_replicas - stopping_replicas, 0) if to_add > 0: logger.info(f"Adding {to_add} replicas " f"to backend '{backend_tag}'.") for _ in range(to_add): replica_tag = "{}#{}".format(backend_tag, get_random_letters()) self._replicas[backend_tag].add( ReplicaState.SHOULD_START, BackendReplica(self._controller_name, self._detached, replica_tag, backend_tag, target_version)) elif delta_replicas < 0: to_remove = -delta_replicas logger.info(f"Removing {to_remove} replicas " f"from backend '{backend_tag}'.") replicas_to_stop = self._replicas[backend_tag].pop( states=[ ReplicaState.SHOULD_START, ReplicaState.STARTING, ReplicaState.RUNNING ], max_replicas=to_remove) for replica in replicas_to_stop: replica.set_should_stop(graceful_shutdown_timeout_s) self._replicas[backend_tag].add(ReplicaState.SHOULD_STOP, replica) return True
def make_request_param(call_method="__call__"): return RequestMetadata(get_random_letters(10), "endpoint", context.TaskContext.Python, call_method=call_method)