async def deploy(
self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, python_methods: List[str],
version: Optional[str],
route_prefix: Optional[str]) -> Tuple[Optional[GoalId], bool]:
if route_prefix is not None:
assert route_prefix.startswith("/")
async with self.write_lock:
backend_info = BackendInfo(actor_def=ray.remote(
create_backend_replica(name,
replica_config.serialized_backend_def)),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id, updating = self.backend_state.deploy_backend(
name, backend_info)
endpoint_info = EndpointInfo(ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods,
legacy=False)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({name: 1.0}))
return goal_id, updating
def create_backend(self, backend_tag: BackendTag,
backend_config: BackendConfig,
replica_config: ReplicaConfig) -> Optional[GoalId]:
# Ensures this method is idempotent.
backend_info = self._backend_metadata.get(backend_tag)
if backend_info is not None:
if (backend_info.backend_config == backend_config
and backend_info.replica_config == replica_config):
return None
backend_replica_class = create_backend_replica(
replica_config.func_or_class)
# Save creator that starts replicas, the arguments to be passed in,
# and the configuration for the backends.
backend_info = BackendInfo(worker_class=backend_replica_class,
backend_config=backend_config,
replica_config=replica_config)
new_goal_id, existing_goal_id = self._set_backend_goal(
backend_tag, backend_info)
# NOTE(edoakes): we must write a checkpoint before starting new
# or pushing the updated config to avoid inconsistent state if we
# crash while making the change.
self._checkpoint()
self._notify_backend_configs_changed()
if existing_goal_id is not None:
self._goal_manager.complete_goal(existing_goal_id)
return new_goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is not None:
assert route_prefix.startswith("/")
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
endpoint_info = EndpointInfo(ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({name: 1.0}))
return goal_id
def backend_info(version: Optional[str] = None,
num_replicas: Optional[int] = 1,
**config_opts) -> BackendInfo:
return BackendInfo(
worker_class=None,
version=version,
backend_config=BackendConfig(num_replicas=num_replicas, **config_opts),
replica_config=ReplicaConfig(lambda x: x))
def generate_mock_backend_info(
num_replicas: Optional[int] = None) -> BackendInfo:
backend_info = BackendInfo(worker_class=lambda x: x,
backend_config=BackendConfig(),
replica_config=ReplicaConfig(lambda x: x))
if num_replicas:
backend_info.backend_config.num_replicas = num_replicas
return backend_info
async def create_backend(
self, backend_tag: BackendTag, backend_config: BackendConfig,
replica_config: ReplicaConfig) -> Optional[GoalId]:
"""Register a new backend under the specified tag."""
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=RESERVED_VERSION_TAG,
backend_config=backend_config,
replica_config=replica_config)
return self.backend_state.deploy_backend(backend_tag, backend_info)
def deploy_backend(
self, backend_tag: BackendTag,
backend_info: BackendInfo) -> Tuple[Optional[GoalId], bool]:
"""Deploy the backend.
If the backend already exists with the same version, this is a no-op
and returns the GoalId corresponding to the existing update if there
is one.
Returns:
GoalId, bool: The GoalId for the client to wait for and whether or
not the backend is being updated.
"""
# Ensures this method is idempotent.
existing_info = self._backend_metadata.get(backend_tag)
if existing_info is not None:
# Redeploying should not reset the deployment's start time.
backend_info.start_time_ms = existing_info.start_time_ms
# Old codepath. We use RESERVED_VERSION_TAG to distinguish that
# we shouldn't use versions at all to determine redeployment
# because `None` is used to indicate always redeploying.
if backend_info.version == RESERVED_VERSION_TAG:
if (existing_info.backend_config == backend_info.backend_config
and existing_info.replica_config
== backend_info.replica_config):
return self._backend_goals.get(backend_tag, None), False
# New codepath: treat version as ground truth for implementation.
elif (existing_info.backend_config == backend_info.backend_config
and backend_info.version is not None
and existing_info.version == backend_info.version):
return self._backend_goals.get(backend_tag, None), False
if backend_tag not in self._replicas:
self._replicas[backend_tag] = ReplicaStateContainer()
new_goal_id, existing_goal_id = self._set_backend_goal(
backend_tag, backend_info)
if backend_tag in self._deleted_backend_metadata:
del self._deleted_backend_metadata[backend_tag]
# NOTE(edoakes): we must write a checkpoint before starting new
# or pushing the updated config to avoid inconsistent state if we
# crash while making the change.
self._checkpoint()
self._notify_backend_configs_changed(backend_tag)
if existing_goal_id is not None:
self._goal_manager.complete_goal(existing_goal_id)
return new_goal_id, True
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
# By default the path prefix is the deployment name.
if replica_config.path_prefix is None:
replica_config.path_prefix = f"/{name}"
# Backend config should be synchronized so the backend worker
# is aware of it.
backend_config.internal_metadata.path_prefix = f"/{name}"
else:
if ("{" in replica_config.path_prefix
or "}" in replica_config.path_prefix):
raise ValueError(
"Wildcard routes are not supported for deployment paths. "
"Please use @serve.ingress with FastAPI instead.")
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
path_prefix = replica_config.path_prefix
if path_prefix.endswith("/"):
path_prefix = path_prefix[:-1]
http_route = path_prefix + WILDCARD_PATH_SUFFIX
http_methods = ALL_HTTP_METHODS
else:
http_route = replica_config.path_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.create_endpoint(name,
http_route,
http_methods,
TrafficPolicy({name: 1.0}),
python_methods=python_methods)
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
# By default the path prefix is the deployment name.
if replica_config.path_prefix is None:
replica_config.path_prefix = f"/{name}"
# Backend config should be synchronized so the backend worker
# is aware of it.
backend_config.internal_metadata.path_prefix = f"/{name}"
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
http_route = f"{replica_config.path_prefix}" + "/{wildcard:path}"
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
http_methods = [
"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS",
"TRACE", "PATCH"
]
else:
http_route = replica_config.path_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(
worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.create_endpoint(
name,
http_route,
http_methods,
TrafficPolicy({
name: 1.0
}),
python_methods=python_methods)
return goal_id
async def create_backend(
self, backend_tag: BackendTag, backend_config: BackendConfig,
replica_config: ReplicaConfig) -> Optional[GoalId]:
"""Register a new backend under the specified tag."""
async with self.write_lock:
backend_info = BackendInfo(actor_def=ray.remote(
create_backend_replica(backend_tag,
replica_config.serialized_backend_def)),
version=RESERVED_VERSION_TAG,
backend_config=backend_config,
replica_config=replica_config)
goal_id, _ = self.backend_state.deploy_backend(
backend_tag, backend_info)
return goal_id
async def deploy(self,
name: str,
backend_config: BackendConfig,
replica_config: ReplicaConfig,
python_methods: List[str],
version: Optional[str],
prev_version: Optional[str],
route_prefix: Optional[str],
deployer_job_id: "Optional[ray._raylet.JobID]" = None
) -> Tuple[Optional[GoalId], bool]:
if route_prefix is not None:
assert route_prefix.startswith("/")
async with self.write_lock:
if prev_version is not None:
existing_backend_info = self.backend_state.get_backend(name)
if (existing_backend_info is None
or not existing_backend_info.version):
raise ValueError(
f"prev_version '{prev_version}' is specified but "
"there is no existing deployment.")
if existing_backend_info.version != prev_version:
raise ValueError(
f"prev_version '{prev_version}' "
"does not match with the existing "
f"version '{existing_backend_info.version}'.")
backend_info = BackendInfo(
actor_def=ray.remote(
create_backend_replica(
name, replica_config.serialized_backend_def)),
version=version,
backend_config=backend_config,
replica_config=replica_config,
deployer_job_id=deployer_job_id,
start_time_ms=int(time.time() * 1000))
goal_id, updating = self.backend_state.deploy_backend(
name, backend_info)
endpoint_info = EndpointInfo(
ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods,
legacy=False)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({
name: 1.0
}))
return goal_id, updating
async def update_backend_config(self, backend_tag: BackendTag,
config_options: BackendConfig) -> GoalId:
"""Set the config for the specified backend."""
async with self.write_lock:
existing_info = self.backend_state.get_backend(backend_tag)
if existing_info is None:
raise ValueError(f"Backend {backend_tag} is not registered.")
backend_info = BackendInfo(
worker_class=existing_info.worker_class,
version=existing_info.version,
backend_config=existing_info.backend_config.copy(
update=config_options.dict(exclude_unset=True)),
replica_config=existing_info.replica_config)
return self.backend_state.deploy_backend(backend_tag, backend_info)
def deploy(
self,
name: str,
backend_config_proto_bytes: bytes,
replica_config: ReplicaConfig,
version: Optional[str],
prev_version: Optional[str],
route_prefix: Optional[str],
deployer_job_id: "Optional[ray._raylet.JobID]" = None
) -> Tuple[Optional[GoalId], bool]:
if route_prefix is not None:
assert route_prefix.startswith("/")
backend_config = BackendConfig.from_proto_bytes(
backend_config_proto_bytes)
if prev_version is not None:
existing_backend_info = self.backend_state_manager.get_backend(
name)
if (existing_backend_info is None
or not existing_backend_info.version):
raise ValueError(
f"prev_version '{prev_version}' is specified but "
"there is no existing deployment.")
if existing_backend_info.version != prev_version:
raise ValueError(f"prev_version '{prev_version}' "
"does not match with the existing "
f"version '{existing_backend_info.version}'.")
backend_info = BackendInfo(actor_def=ray.remote(
create_replica_wrapper(name,
replica_config.serialized_backend_def)),
version=version,
backend_config=backend_config,
replica_config=replica_config,
deployer_job_id=deployer_job_id,
start_time_ms=int(time.time() * 1000))
# TODO(architkulkarni): When a deployment is redeployed, even if
# the only change was num_replicas, the start_time_ms is refreshed.
# This is probably not the desired behavior for an autoscaling
# deployment, which redeploys very often to change num_replicas.
goal_id, updating = self.backend_state_manager.deploy_backend(
name, backend_info)
endpoint_info = EndpointInfo(route=route_prefix)
self.endpoint_state.update_endpoint(name, endpoint_info)
return goal_id, updating
async def update_backend_config(self, backend_tag: BackendTag,
config_options: BackendConfig) -> GoalId:
"""Set the config for the specified backend."""
async with self.write_lock:
existing_info = self.backend_state.get_backend(backend_tag)
if existing_info is None:
raise ValueError(f"Backend {backend_tag} is not registered.")
backend_info = BackendInfo(
actor_def=existing_info.actor_def,
version=existing_info.version,
backend_config=existing_info.backend_config.copy(
update=config_options.dict(exclude_unset=True)),
replica_config=existing_info.replica_config,
deployer_job_id=existing_info.deployer_job_id,
start_time_ms=existing_info.start_time_ms)
goal_id, _ = self.backend_state.deploy_backend(
backend_tag, backend_info)
return goal_id
def deploy_backend(self,
backend_tag: BackendTag,
backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str] = None) -> Optional[GoalId]:
# Ensures this method is idempotent.
backend_info = self._backend_metadata.get(backend_tag)
if backend_info is not None:
# Old codepath.
if version is None:
if (backend_info.backend_config == backend_config
and backend_info.replica_config == replica_config):
return self._backend_goals.get(backend_tag, None)
# New codepath: treat version as ground truth for implementation.
else:
if (backend_info.backend_config == backend_config
and self._target_versions[backend_tag] == version):
return self._backend_goals.get(backend_tag, None)
if backend_tag not in self._replicas:
self._replicas[backend_tag] = ReplicaStateContainer()
backend_replica_class = create_backend_replica(
replica_config.backend_def)
# Save creator that starts replicas, the arguments to be passed in,
# and the configuration for the backends.
backend_info = BackendInfo(worker_class=backend_replica_class,
backend_config=backend_config,
replica_config=replica_config)
new_goal_id, existing_goal_id = self._set_backend_goal(
backend_tag, backend_info, version)
# NOTE(edoakes): we must write a checkpoint before starting new
# or pushing the updated config to avoid inconsistent state if we
# crash while making the change.
self._checkpoint()
self._notify_backend_configs_changed(backend_tag)
if existing_goal_id is not None:
self._goal_manager.complete_goal(existing_goal_id)
return new_goal_id
async def create_backend(self, backend_tag: BackendTag,
backend_config: BackendConfig,
replica_config: ReplicaConfig) -> UUID:
"""Register a new backend under the specified tag."""
async with self.write_lock:
# Ensures this method is idempotent.
backend_info = self.backend_state.get_backend(backend_tag)
if backend_info is not None:
if (backend_info.backend_config == backend_config
and backend_info.replica_config == replica_config):
return
backend_replica = create_backend_replica(
replica_config.func_or_class)
# Save creator that starts replicas, the arguments to be passed in,
# and the configuration for the backends.
backend_info = BackendInfo(
worker_class=backend_replica,
backend_config=backend_config,
replica_config=replica_config)
return_uuid = self._create_event_with_result({
backend_tag: backend_info
})
await self.set_backend_goal(backend_tag, backend_info, return_uuid)
try:
# This call should be to run control loop
self.backend_state.scale_backend_replicas(
backend_tag, backend_config.num_replicas)
except RayServeException as e:
del self.backend_state.backends[backend_tag]
raise e
# NOTE(edoakes): we must write a checkpoint before starting new
# or pushing the updated config to avoid inconsistent state if we
# crash while making the change.
self._checkpoint()
self.notify_backend_configs_changed()
return return_uuid
async def create_backend(
self,
backend_tag: BackendTag,
backend_config: BackendConfig,
replica_config: ReplicaConfig,
deployer_job_id: Optional["Optional[ray._raylet.JobID]"] = None
) -> Optional[GoalId]:
"""Register a new backend under the specified tag."""
async with self.write_lock:
backend_info = BackendInfo(actor_def=ray.remote(
create_backend_replica(backend_tag,
replica_config.serialized_backend_def)),
version=RESERVED_VERSION_TAG,
backend_config=backend_config,
replica_config=replica_config,
start_time_ms=int(time.time() * 1000),
deployer_job_id=deployer_job_id)
goal_id, _ = self.backend_state.deploy_backend(
backend_tag, backend_info)
return goal_id
def deploy(self,
backend_info: BackendInfo) -> Tuple[Optional[GoalId], bool]:
"""Deploy the backend.
If the backend already exists with the same version, this is a no-op
and returns the GoalId corresponding to the existing update if there
is one.
Returns:
GoalId, bool: The GoalId for the client to wait for and whether or
not the backend is being updated.
"""
# Ensures this method is idempotent.
existing_info = self._target_info
if existing_info is not None:
# Redeploying should not reset the deployment's start time.
backend_info.start_time_ms = existing_info.start_time_ms
if (existing_info.backend_config == backend_info.backend_config
and backend_info.version is not None
and existing_info.version == backend_info.version):
return self._curr_goal, False
# Keep a copy of previous backend info in case goal failed to
# complete to initiate rollback.
self._rollback_info = self._target_info
# Reset constructor retry counter.
self._replica_constructor_retry_counter = 0
new_goal_id, existing_goal_id = self._set_backend_goal(backend_info)
# NOTE(edoakes): we must write a checkpoint before starting new
# or pushing the updated config to avoid inconsistent state if we
# crash while making the change.
self._checkpoint_fn()
self._notify_backend_configs_changed()
if existing_goal_id is not None:
self._goal_manager.complete_goal(existing_goal_id)
return new_goal_id, True
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is None:
route_prefix = f"/{name}"
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
if route_prefix.endswith("/"):
route_prefix = route_prefix[:-1]
http_route = route_prefix + WILDCARD_PATH_SUFFIX
http_methods = ALL_HTTP_METHODS
else:
http_route = route_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.update_endpoint(name,
http_route,
http_methods,
TrafficPolicy({name: 1.0}),
python_methods=python_methods)
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is not None:
assert route_prefix.startswith("/")
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
http_methods = ALL_HTTP_METHODS
else:
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(
worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
endpoint_info = EndpointInfo(
http_methods,
route=route_prefix,
python_methods=python_methods)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({
name: 1.0
}))
return goal_id
