def _scale_replicas(self, backends: Dict[BackendTag, BackendInfo],
backend_tag: BackendTag, num_replicas: int) -> None:
"""Scale the given backend to the number of replicas.
NOTE: this does not actually start or stop the replicas, but instead
adds the intention to start/stop them to self.workers_to_start and
self.workers_to_stop. The caller is responsible for then first writing
a checkpoint and then actually starting/stopping the intended replicas.
This avoids inconsistencies with starting/stopping a worker and then
crashing before writing a checkpoint.
"""
logger.debug("Scaling backend '{}' to {} replicas".format(
backend_tag, num_replicas))
assert (
backend_tag
in backends), "Backend {} is not registered.".format(backend_tag)
assert num_replicas >= 0, ("Number of replicas must be"
" greater than or equal to 0.")
current_num_replicas = len(self.replicas[backend_tag])
delta_num_replicas = num_replicas - current_num_replicas
backend_info = backends[backend_tag]
if delta_num_replicas > 0:
can_schedule = try_schedule_resources_on_nodes(requirements=[
backend_info.replica_config.resource_dict
for _ in range(delta_num_replicas)
])
if _RESOURCE_CHECK_ENABLED and not all(can_schedule):
num_possible = sum(can_schedule)
raise RayServeException(
"Cannot scale backend {} to {} replicas. Ray Serve tried "
"to add {} replicas but the resources only allows {} "
"to be added. To fix this, consider scaling to replica to "
"{} or add more resources to the cluster. You can check "
"avaiable resources with ray.nodes().".format(
backend_tag, num_replicas, delta_num_replicas,
num_possible, current_num_replicas + num_possible))
logger.debug("Adding {} replicas to backend {}".format(
delta_num_replicas, backend_tag))
for _ in range(delta_num_replicas):
replica_tag = "{}#{}".format(backend_tag, get_random_letters())
self.replicas_to_start[backend_tag].append(replica_tag)
elif delta_num_replicas < 0:
logger.debug("Removing {} replicas from backend '{}'".format(
-delta_num_replicas, backend_tag))
assert len(self.replicas[backend_tag]) >= delta_num_replicas
for _ in range(-delta_num_replicas):
replica_tag = self.replicas[backend_tag].pop()
if len(self.replicas[backend_tag]) == 0:
del self.replicas[backend_tag]
del self.workers[backend_tag][replica_tag]
if len(self.workers[backend_tag]) == 0:
del self.workers[backend_tag]
self.replicas_to_stop[backend_tag].append(replica_tag)
def test_mock_scheduler():
ray_nodes = [{
"NodeID": "AAA",
"Alive": True,
"Resources": {
"CPU": 2.0,
"GPU": 2.0
}
}, {
"NodeID": "BBB",
"Alive": True,
"Resources": {
"CPU": 4.0,
}
}]
assert try_schedule_resources_on_nodes(
[
{
"CPU": 2,
"GPU": 2
}, # node 1
{
"CPU": 4
} # node 2
],
deepcopy(ray_nodes)) == [True, True]
assert try_schedule_resources_on_nodes([
{
"CPU": 100
},
{
"GPU": 1
},
], deepcopy(ray_nodes)) == [False, True]
assert try_schedule_resources_on_nodes(
[
{
"CPU": 6
}, # Equals to the sum of cpus but shouldn't be scheduable.
],
deepcopy(ray_nodes)) == [False]
def _scale_backend_replicas(
self,
backend_tag: BackendTag,
num_replicas: int,
) -> bool:
"""Scale the given backend to the number of replicas.
NOTE: this does not actually start or stop the replicas, but instead
adds them to ReplicaState.SHOULD_START or ReplicaState.SHOULD_STOP.
The caller is responsible for then first writing a checkpoint and then
actually starting/stopping the intended replicas. This avoids
inconsistencies with starting/stopping a replica and then crashing
before writing a checkpoint.
"""
logger.debug("Scaling backend '{}' to {} replicas".format(
backend_tag, num_replicas))
assert (backend_tag in self._backend_metadata
), "Backend {} is not registered.".format(backend_tag)
assert num_replicas >= 0, ("Number of replicas must be"
" greater than or equal to 0.")
current_num_replicas = sum([
len(self._replicas[backend_tag][ReplicaState.SHOULD_START]),
len(self._replicas[backend_tag][ReplicaState.STARTING]),
len(self._replicas[backend_tag][ReplicaState.RUNNING]),
])
delta_num_replicas = num_replicas - current_num_replicas
backend_info: BackendInfo = self._backend_metadata[backend_tag]
if delta_num_replicas == 0:
return False
elif delta_num_replicas > 0:
can_schedule = try_schedule_resources_on_nodes(requirements=[
backend_info.replica_config.resource_dict
for _ in range(delta_num_replicas)
])
if _RESOURCE_CHECK_ENABLED and not all(can_schedule):
num_possible = sum(can_schedule)
logger.error(
"Cannot scale backend {} to {} replicas. Ray Serve tried "
"to add {} replicas but the resources only allows {} "
"to be added. This is not a problem if the cluster is "
"autoscaling. To fix this, consider scaling to replica to "
"{} or add more resources to the cluster. You can check "
"avaiable resources with ray.nodes().".format(
backend_tag, num_replicas, delta_num_replicas,
num_possible, current_num_replicas + num_possible))
logger.debug("Adding {} replicas to backend {}".format(
delta_num_replicas, backend_tag))
for _ in range(delta_num_replicas):
replica_tag = "{}#{}".format(backend_tag, get_random_letters())
self._replicas[backend_tag][ReplicaState.SHOULD_START].append(
BackendReplica(self._controller_name, self._detached,
replica_tag, backend_tag))
elif delta_num_replicas < 0:
logger.debug("Removing {} replicas from backend '{}'".format(
-delta_num_replicas, backend_tag))
assert self._target_replicas[backend_tag] >= delta_num_replicas
for _ in range(-delta_num_replicas):
replica_state_dict = self._replicas[backend_tag]
list_to_use = replica_state_dict[ReplicaState.SHOULD_START] \
or replica_state_dict[ReplicaState.STARTING] \
or replica_state_dict[ReplicaState.RUNNING]
assert len(list_to_use), replica_state_dict
replica_to_stop = list_to_use.pop()
graceful_timeout_s = (backend_info.backend_config.
experimental_graceful_shutdown_timeout_s)
replica_to_stop.set_should_stop(graceful_timeout_s)
self._replicas[backend_tag][ReplicaState.SHOULD_STOP].append(
replica_to_stop)
return True
