async def deploy(
self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, python_methods: List[str],
version: Optional[str],
route_prefix: Optional[str]) -> Tuple[Optional[GoalId], bool]:
if route_prefix is not None:
assert route_prefix.startswith("/")
async with self.write_lock:
backend_info = BackendInfo(actor_def=ray.remote(
create_backend_replica(name,
replica_config.serialized_backend_def)),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id, updating = self.backend_state.deploy_backend(
name, backend_info)
endpoint_info = EndpointInfo(ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods,
legacy=False)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({name: 1.0}))
return goal_id, updating
async def _set_traffic(self, endpoint_name: str,
traffic_dict: Dict[str, float]) -> UUID:
if endpoint_name not in self.endpoint_state.get_endpoints():
raise ValueError("Attempted to assign traffic for an endpoint '{}'"
" that is not registered.".format(endpoint_name))
assert isinstance(traffic_dict,
dict), "Traffic policy must be a dictionary."
for backend in traffic_dict:
if self.backend_state.get_backend(backend) is None:
raise ValueError(
"Attempted to assign traffic to a backend '{}' that "
"is not registered.".format(backend))
traffic_policy = TrafficPolicy(traffic_dict)
self.endpoint_state.traffic_policies[endpoint_name] = traffic_policy
return_uuid = self._create_event_with_result(
{endpoint_name: traffic_policy})
# NOTE(edoakes): we must write a checkpoint before pushing the
# update to avoid inconsistent state if we crash after pushing the
# update.
self._checkpoint()
self.notify_traffic_policies_changed()
self.set_goal_id(return_uuid)
return return_uuid
async def create_endpoint(
self,
endpoint: str,
traffic_dict: Dict[str, float],
route: Optional[str],
methods: Set[str],
) -> None:
"""Create a new endpoint with the specified route and methods.
If the route is None, this is a "headless" endpoint that will not
be exposed over HTTP and can only be accessed via a handle.
"""
async with self.write_lock:
self._validate_traffic_dict(traffic_dict)
logger.info(
"Registering route '{}' to endpoint '{}' with methods '{}'.".
format(route, endpoint, methods))
self.endpoint_state.create_endpoint(
endpoint, EndpointInfo(methods, route=route),
TrafficPolicy(traffic_dict))
# TODO(simon): Use GoalID mechanism for this so client can check for
# goal id and http_state complete the goal id.
await self.http_state.ensure_http_route_exists(endpoint, timeout_s=30)
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is not None:
assert route_prefix.startswith("/")
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
endpoint_info = EndpointInfo(ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({name: 1.0}))
return goal_id
def _set_traffic(self, endpoint_name: str,
traffic_dict: Dict[str, float]) -> UUID:
for backend in traffic_dict:
if self.backend_state.get_backend(backend) is None:
raise ValueError(
"Attempted to assign traffic to a backend '{}' that "
"is not registered.".format(backend))
self.endpoint_state.set_traffic_policy(endpoint_name,
TrafficPolicy(traffic_dict))
async def set_traffic(self, endpoint: str,
traffic_dict: Dict[str, float]) -> None:
"""Sets the traffic policy for the specified endpoint."""
async with self.write_lock:
self._validate_traffic_dict(traffic_dict)
logger.info("Setting traffic for endpoint "
f"'{endpoint}' to '{traffic_dict}'.")
self.endpoint_state.set_traffic_policy(endpoint,
TrafficPolicy(traffic_dict))
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
# By default the path prefix is the deployment name.
if replica_config.path_prefix is None:
replica_config.path_prefix = f"/{name}"
# Backend config should be synchronized so the backend worker
# is aware of it.
backend_config.internal_metadata.path_prefix = f"/{name}"
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
http_route = f"{replica_config.path_prefix}" + "/{wildcard:path}"
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
http_methods = [
"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS",
"TRACE", "PATCH"
]
else:
http_route = replica_config.path_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
if version is None:
version = RESERVED_VERSION_TAG
else:
if version == RESERVED_VERSION_TAG:
# TODO(edoakes): this is unlikely to ever be hit, but it's
# still ugly and should be removed once the old codepath
# can be deleted.
raise ValueError(
f"Version {RESERVED_VERSION_TAG} is reserved and "
"cannot be used by applications.")
goal_id = self.backend_state.deploy_backend(
name, backend_config, replica_config, version)
self.endpoint_state.create_endpoint(
name,
http_route,
http_methods,
TrafficPolicy({
name: 1.0
}),
python_methods=python_methods)
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
# By default the path prefix is the deployment name.
if replica_config.path_prefix is None:
replica_config.path_prefix = f"/{name}"
# Backend config should be synchronized so the backend worker
# is aware of it.
backend_config.internal_metadata.path_prefix = f"/{name}"
else:
if ("{" in replica_config.path_prefix
or "}" in replica_config.path_prefix):
raise ValueError(
"Wildcard routes are not supported for deployment paths. "
"Please use @serve.ingress with FastAPI instead.")
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
path_prefix = replica_config.path_prefix
if path_prefix.endswith("/"):
path_prefix = path_prefix[:-1]
http_route = path_prefix + WILDCARD_PATH_SUFFIX
http_methods = ALL_HTTP_METHODS
else:
http_route = replica_config.path_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.create_endpoint(name,
http_route,
http_methods,
TrafficPolicy({name: 1.0}),
python_methods=python_methods)
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
# By default the path prefix is the deployment name.
if replica_config.path_prefix is None:
replica_config.path_prefix = f"/{name}"
# Backend config should be synchronized so the backend worker
# is aware of it.
backend_config.internal_metadata.path_prefix = f"/{name}"
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
http_route = f"{replica_config.path_prefix}" + "/{wildcard:path}"
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
http_methods = [
"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS",
"TRACE", "PATCH"
]
else:
http_route = replica_config.path_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(
worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.create_endpoint(
name,
http_route,
http_methods,
TrafficPolicy({
name: 1.0
}),
python_methods=python_methods)
return goal_id
async def deploy(self,
name: str,
backend_config: BackendConfig,
replica_config: ReplicaConfig,
python_methods: List[str],
version: Optional[str],
prev_version: Optional[str],
route_prefix: Optional[str],
deployer_job_id: "Optional[ray._raylet.JobID]" = None
) -> Tuple[Optional[GoalId], bool]:
if route_prefix is not None:
assert route_prefix.startswith("/")
async with self.write_lock:
if prev_version is not None:
existing_backend_info = self.backend_state.get_backend(name)
if (existing_backend_info is None
or not existing_backend_info.version):
raise ValueError(
f"prev_version '{prev_version}' is specified but "
"there is no existing deployment.")
if existing_backend_info.version != prev_version:
raise ValueError(
f"prev_version '{prev_version}' "
"does not match with the existing "
f"version '{existing_backend_info.version}'.")
backend_info = BackendInfo(
actor_def=ray.remote(
create_backend_replica(
name, replica_config.serialized_backend_def)),
version=version,
backend_config=backend_config,
replica_config=replica_config,
deployer_job_id=deployer_job_id,
start_time_ms=int(time.time() * 1000))
goal_id, updating = self.backend_state.deploy_backend(
name, backend_info)
endpoint_info = EndpointInfo(
ALL_HTTP_METHODS,
route=route_prefix,
python_methods=python_methods,
legacy=False)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({
name: 1.0
}))
return goal_id, updating
async def create_endpoint(self, endpoint: str,
traffic_dict: Dict[str, float], route,
methods: List[str]) -> UUID:
"""Create a new endpoint with the specified route and methods.
If the route is None, this is a "headless" endpoint that will not
be exposed over HTTP and can only be accessed via a handle.
"""
async with self.write_lock:
self._validate_traffic_dict(traffic_dict)
logger.info(
"Registering route '{}' to endpoint '{}' with methods '{}'.".
format(route, endpoint, methods))
self.endpoint_state.create_endpoint(endpoint, route, methods,
TrafficPolicy(traffic_dict))
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is None:
route_prefix = f"/{name}"
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
# {wildcard:path} is used so HTTPProxy's Starlette router can match
# arbitrary path.
if route_prefix.endswith("/"):
route_prefix = route_prefix[:-1]
http_route = route_prefix + WILDCARD_PATH_SUFFIX
http_methods = ALL_HTTP_METHODS
else:
http_route = route_prefix
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
self.endpoint_state.update_endpoint(name,
http_route,
http_methods,
TrafficPolicy({name: 1.0}),
python_methods=python_methods)
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig,
version: Optional[str]) -> Optional[GoalId]:
"""TODO."""
async with self.write_lock:
if version is None:
version = RESERVED_VERSION_TAG
else:
if version == RESERVED_VERSION_TAG:
# TODO(edoakes): this is unlikely to ever be hit, but it's
# still ugly and should be removed once the old codepath
# can be deleted.
raise ValueError(
f"Version {RESERVED_VERSION_TAG} is reserved and "
"cannot be used by applications.")
goal_id = self.backend_state.deploy_backend(
name, backend_config, replica_config, version)
self.endpoint_state.create_endpoint(name, f"/{name}",
["GET", "POST"],
TrafficPolicy({name: 1.0}))
return goal_id
async def deploy(self, name: str, backend_config: BackendConfig,
replica_config: ReplicaConfig, version: Optional[str],
route_prefix: Optional[str]) -> Optional[GoalId]:
if route_prefix is not None:
assert route_prefix.startswith("/")
if replica_config.is_asgi_app:
# When the backend is asgi application, we want to proxy it
# with a prefixed path as well as proxy all HTTP methods.
http_methods = ALL_HTTP_METHODS
else:
# Generic endpoint should support a limited subset of HTTP methods.
http_methods = ["GET", "POST"]
python_methods = []
if inspect.isclass(replica_config.backend_def):
for method_name, _ in inspect.getmembers(
replica_config.backend_def, inspect.isfunction):
python_methods.append(method_name)
async with self.write_lock:
backend_info = BackendInfo(
worker_class=create_backend_replica(
replica_config.backend_def),
version=version,
backend_config=backend_config,
replica_config=replica_config)
goal_id = self.backend_state.deploy_backend(name, backend_info)
endpoint_info = EndpointInfo(
http_methods,
route=route_prefix,
python_methods=python_methods)
self.endpoint_state.update_endpoint(name, endpoint_info,
TrafficPolicy({
name: 1.0
}))
return goal_id