def _scale_replicas(self, backends: Dict[BackendTag, BackendInfo], backend_tag: BackendTag, num_replicas: int) -> None: """Scale the given backend to the number of replicas. NOTE: this does not actually start or stop the replicas, but instead adds the intention to start/stop them to self.workers_to_start and self.workers_to_stop. The caller is responsible for then first writing a checkpoint and then actually starting/stopping the intended replicas. This avoids inconsistencies with starting/stopping a worker and then crashing before writing a checkpoint. """ logger.debug("Scaling backend '{}' to {} replicas".format( backend_tag, num_replicas)) assert ( backend_tag in backends), "Backend {} is not registered.".format(backend_tag) assert num_replicas >= 0, ("Number of replicas must be" " greater than or equal to 0.") current_num_replicas = len(self.replicas[backend_tag]) delta_num_replicas = num_replicas - current_num_replicas backend_info = backends[backend_tag] if delta_num_replicas > 0: can_schedule = try_schedule_resources_on_nodes(requirements=[ backend_info.replica_config.resource_dict for _ in range(delta_num_replicas) ]) if _RESOURCE_CHECK_ENABLED and not all(can_schedule): num_possible = sum(can_schedule) raise RayServeException( "Cannot scale backend {} to {} replicas. Ray Serve tried " "to add {} replicas but the resources only allows {} " "to be added. To fix this, consider scaling to replica to " "{} or add more resources to the cluster. You can check " "avaiable resources with ray.nodes().".format( backend_tag, num_replicas, delta_num_replicas, num_possible, current_num_replicas + num_possible)) logger.debug("Adding {} replicas to backend {}".format( delta_num_replicas, backend_tag)) for _ in range(delta_num_replicas): replica_tag = "{}#{}".format(backend_tag, get_random_letters()) self.replicas_to_start[backend_tag].append(replica_tag) elif delta_num_replicas < 0: logger.debug("Removing {} replicas from backend '{}'".format( -delta_num_replicas, backend_tag)) assert len(self.replicas[backend_tag]) >= delta_num_replicas for _ in range(-delta_num_replicas): replica_tag = self.replicas[backend_tag].pop() if len(self.replicas[backend_tag]) == 0: del self.replicas[backend_tag] del self.workers[backend_tag][replica_tag] if len(self.workers[backend_tag]) == 0: del self.workers[backend_tag] self.replicas_to_stop[backend_tag].append(replica_tag)
def test_mock_scheduler(): ray_nodes = [{ "NodeID": "AAA", "Alive": True, "Resources": { "CPU": 2.0, "GPU": 2.0 } }, { "NodeID": "BBB", "Alive": True, "Resources": { "CPU": 4.0, } }] assert try_schedule_resources_on_nodes( [ { "CPU": 2, "GPU": 2 }, # node 1 { "CPU": 4 } # node 2 ], deepcopy(ray_nodes)) == [True, True] assert try_schedule_resources_on_nodes([ { "CPU": 100 }, { "GPU": 1 }, ], deepcopy(ray_nodes)) == [False, True] assert try_schedule_resources_on_nodes( [ { "CPU": 6 }, # Equals to the sum of cpus but shouldn't be scheduable. ], deepcopy(ray_nodes)) == [False]
def _scale_backend_replicas( self, backend_tag: BackendTag, num_replicas: int, ) -> bool: """Scale the given backend to the number of replicas. NOTE: this does not actually start or stop the replicas, but instead adds them to ReplicaState.SHOULD_START or ReplicaState.SHOULD_STOP. The caller is responsible for then first writing a checkpoint and then actually starting/stopping the intended replicas. This avoids inconsistencies with starting/stopping a replica and then crashing before writing a checkpoint. """ logger.debug("Scaling backend '{}' to {} replicas".format( backend_tag, num_replicas)) assert (backend_tag in self._backend_metadata ), "Backend {} is not registered.".format(backend_tag) assert num_replicas >= 0, ("Number of replicas must be" " greater than or equal to 0.") current_num_replicas = sum([ len(self._replicas[backend_tag][ReplicaState.SHOULD_START]), len(self._replicas[backend_tag][ReplicaState.STARTING]), len(self._replicas[backend_tag][ReplicaState.RUNNING]), ]) delta_num_replicas = num_replicas - current_num_replicas backend_info: BackendInfo = self._backend_metadata[backend_tag] if delta_num_replicas == 0: return False elif delta_num_replicas > 0: can_schedule = try_schedule_resources_on_nodes(requirements=[ backend_info.replica_config.resource_dict for _ in range(delta_num_replicas) ]) if _RESOURCE_CHECK_ENABLED and not all(can_schedule): num_possible = sum(can_schedule) logger.error( "Cannot scale backend {} to {} replicas. Ray Serve tried " "to add {} replicas but the resources only allows {} " "to be added. This is not a problem if the cluster is " "autoscaling. To fix this, consider scaling to replica to " "{} or add more resources to the cluster. You can check " "avaiable resources with ray.nodes().".format( backend_tag, num_replicas, delta_num_replicas, num_possible, current_num_replicas + num_possible)) logger.debug("Adding {} replicas to backend {}".format( delta_num_replicas, backend_tag)) for _ in range(delta_num_replicas): replica_tag = "{}#{}".format(backend_tag, get_random_letters()) self._replicas[backend_tag][ReplicaState.SHOULD_START].append( BackendReplica(self._controller_name, self._detached, replica_tag, backend_tag)) elif delta_num_replicas < 0: logger.debug("Removing {} replicas from backend '{}'".format( -delta_num_replicas, backend_tag)) assert self._target_replicas[backend_tag] >= delta_num_replicas for _ in range(-delta_num_replicas): replica_state_dict = self._replicas[backend_tag] list_to_use = replica_state_dict[ReplicaState.SHOULD_START] \ or replica_state_dict[ReplicaState.STARTING] \ or replica_state_dict[ReplicaState.RUNNING] assert len(list_to_use), replica_state_dict replica_to_stop = list_to_use.pop() graceful_timeout_s = (backend_info.backend_config. experimental_graceful_shutdown_timeout_s) replica_to_stop.set_should_stop(graceful_timeout_s) self._replicas[backend_tag][ReplicaState.SHOULD_STOP].append( replica_to_stop) return True