async def deploy( self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, python_methods: List[str], version: Optional[str], route_prefix: Optional[str]) -> Tuple[Optional[GoalId], bool]: if route_prefix is not None: assert route_prefix.startswith("/") async with self.write_lock: backend_info = BackendInfo(actor_def=ray.remote( create_backend_replica(name, replica_config.serialized_backend_def)), version=version, backend_config=backend_config, replica_config=replica_config) goal_id, updating = self.backend_state.deploy_backend( name, backend_info) endpoint_info = EndpointInfo(ALL_HTTP_METHODS, route=route_prefix, python_methods=python_methods, legacy=False) self.endpoint_state.update_endpoint(name, endpoint_info, TrafficPolicy({name: 1.0})) return goal_id, updating
async def _set_traffic(self, endpoint_name: str, traffic_dict: Dict[str, float]) -> UUID: if endpoint_name not in self.endpoint_state.get_endpoints(): raise ValueError("Attempted to assign traffic for an endpoint '{}'" " that is not registered.".format(endpoint_name)) assert isinstance(traffic_dict, dict), "Traffic policy must be a dictionary." for backend in traffic_dict: if self.backend_state.get_backend(backend) is None: raise ValueError( "Attempted to assign traffic to a backend '{}' that " "is not registered.".format(backend)) traffic_policy = TrafficPolicy(traffic_dict) self.endpoint_state.traffic_policies[endpoint_name] = traffic_policy return_uuid = self._create_event_with_result( {endpoint_name: traffic_policy}) # NOTE(edoakes): we must write a checkpoint before pushing the # update to avoid inconsistent state if we crash after pushing the # update. self._checkpoint() self.notify_traffic_policies_changed() self.set_goal_id(return_uuid) return return_uuid
async def create_endpoint( self, endpoint: str, traffic_dict: Dict[str, float], route: Optional[str], methods: Set[str], ) -> None: """Create a new endpoint with the specified route and methods. If the route is None, this is a "headless" endpoint that will not be exposed over HTTP and can only be accessed via a handle. """ async with self.write_lock: self._validate_traffic_dict(traffic_dict) logger.info( "Registering route '{}' to endpoint '{}' with methods '{}'.". format(route, endpoint, methods)) self.endpoint_state.create_endpoint( endpoint, EndpointInfo(methods, route=route), TrafficPolicy(traffic_dict)) # TODO(simon): Use GoalID mechanism for this so client can check for # goal id and http_state complete the goal id. await self.http_state.ensure_http_route_exists(endpoint, timeout_s=30)
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str], route_prefix: Optional[str]) -> Optional[GoalId]: if route_prefix is not None: assert route_prefix.startswith("/") python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: backend_info = BackendInfo(worker_class=create_backend_replica( replica_config.backend_def), version=version, backend_config=backend_config, replica_config=replica_config) goal_id = self.backend_state.deploy_backend(name, backend_info) endpoint_info = EndpointInfo(ALL_HTTP_METHODS, route=route_prefix, python_methods=python_methods) self.endpoint_state.update_endpoint(name, endpoint_info, TrafficPolicy({name: 1.0})) return goal_id
def _set_traffic(self, endpoint_name: str, traffic_dict: Dict[str, float]) -> UUID: for backend in traffic_dict: if self.backend_state.get_backend(backend) is None: raise ValueError( "Attempted to assign traffic to a backend '{}' that " "is not registered.".format(backend)) self.endpoint_state.set_traffic_policy(endpoint_name, TrafficPolicy(traffic_dict))
async def set_traffic(self, endpoint: str, traffic_dict: Dict[str, float]) -> None: """Sets the traffic policy for the specified endpoint.""" async with self.write_lock: self._validate_traffic_dict(traffic_dict) logger.info("Setting traffic for endpoint " f"'{endpoint}' to '{traffic_dict}'.") self.endpoint_state.set_traffic_policy(endpoint, TrafficPolicy(traffic_dict))
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str]) -> Optional[GoalId]: # By default the path prefix is the deployment name. if replica_config.path_prefix is None: replica_config.path_prefix = f"/{name}" # Backend config should be synchronized so the backend worker # is aware of it. backend_config.internal_metadata.path_prefix = f"/{name}" if replica_config.is_asgi_app: # When the backend is asgi application, we want to proxy it # with a prefixed path as well as proxy all HTTP methods. # {wildcard:path} is used so HTTPProxy's Starlette router can match # arbitrary path. http_route = f"{replica_config.path_prefix}" + "/{wildcard:path}" # https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods http_methods = [ "GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH" ] else: http_route = replica_config.path_prefix # Generic endpoint should support a limited subset of HTTP methods. http_methods = ["GET", "POST"] python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: if version is None: version = RESERVED_VERSION_TAG else: if version == RESERVED_VERSION_TAG: # TODO(edoakes): this is unlikely to ever be hit, but it's # still ugly and should be removed once the old codepath # can be deleted. raise ValueError( f"Version {RESERVED_VERSION_TAG} is reserved and " "cannot be used by applications.") goal_id = self.backend_state.deploy_backend( name, backend_config, replica_config, version) self.endpoint_state.create_endpoint( name, http_route, http_methods, TrafficPolicy({ name: 1.0 }), python_methods=python_methods) return goal_id
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str]) -> Optional[GoalId]: # By default the path prefix is the deployment name. if replica_config.path_prefix is None: replica_config.path_prefix = f"/{name}" # Backend config should be synchronized so the backend worker # is aware of it. backend_config.internal_metadata.path_prefix = f"/{name}" else: if ("{" in replica_config.path_prefix or "}" in replica_config.path_prefix): raise ValueError( "Wildcard routes are not supported for deployment paths. " "Please use @serve.ingress with FastAPI instead.") if replica_config.is_asgi_app: # When the backend is asgi application, we want to proxy it # with a prefixed path as well as proxy all HTTP methods. # {wildcard:path} is used so HTTPProxy's Starlette router can match # arbitrary path. path_prefix = replica_config.path_prefix if path_prefix.endswith("/"): path_prefix = path_prefix[:-1] http_route = path_prefix + WILDCARD_PATH_SUFFIX http_methods = ALL_HTTP_METHODS else: http_route = replica_config.path_prefix # Generic endpoint should support a limited subset of HTTP methods. http_methods = ["GET", "POST"] python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: backend_info = BackendInfo(worker_class=create_backend_replica( replica_config.backend_def), version=version, backend_config=backend_config, replica_config=replica_config) goal_id = self.backend_state.deploy_backend(name, backend_info) self.endpoint_state.create_endpoint(name, http_route, http_methods, TrafficPolicy({name: 1.0}), python_methods=python_methods) return goal_id
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str]) -> Optional[GoalId]: # By default the path prefix is the deployment name. if replica_config.path_prefix is None: replica_config.path_prefix = f"/{name}" # Backend config should be synchronized so the backend worker # is aware of it. backend_config.internal_metadata.path_prefix = f"/{name}" if replica_config.is_asgi_app: # When the backend is asgi application, we want to proxy it # with a prefixed path as well as proxy all HTTP methods. # {wildcard:path} is used so HTTPProxy's Starlette router can match # arbitrary path. http_route = f"{replica_config.path_prefix}" + "/{wildcard:path}" # https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods http_methods = [ "GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH" ] else: http_route = replica_config.path_prefix # Generic endpoint should support a limited subset of HTTP methods. http_methods = ["GET", "POST"] python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: backend_info = BackendInfo( worker_class=create_backend_replica( replica_config.backend_def), version=version, backend_config=backend_config, replica_config=replica_config) goal_id = self.backend_state.deploy_backend(name, backend_info) self.endpoint_state.create_endpoint( name, http_route, http_methods, TrafficPolicy({ name: 1.0 }), python_methods=python_methods) return goal_id
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, python_methods: List[str], version: Optional[str], prev_version: Optional[str], route_prefix: Optional[str], deployer_job_id: "Optional[ray._raylet.JobID]" = None ) -> Tuple[Optional[GoalId], bool]: if route_prefix is not None: assert route_prefix.startswith("/") async with self.write_lock: if prev_version is not None: existing_backend_info = self.backend_state.get_backend(name) if (existing_backend_info is None or not existing_backend_info.version): raise ValueError( f"prev_version '{prev_version}' is specified but " "there is no existing deployment.") if existing_backend_info.version != prev_version: raise ValueError( f"prev_version '{prev_version}' " "does not match with the existing " f"version '{existing_backend_info.version}'.") backend_info = BackendInfo( actor_def=ray.remote( create_backend_replica( name, replica_config.serialized_backend_def)), version=version, backend_config=backend_config, replica_config=replica_config, deployer_job_id=deployer_job_id, start_time_ms=int(time.time() * 1000)) goal_id, updating = self.backend_state.deploy_backend( name, backend_info) endpoint_info = EndpointInfo( ALL_HTTP_METHODS, route=route_prefix, python_methods=python_methods, legacy=False) self.endpoint_state.update_endpoint(name, endpoint_info, TrafficPolicy({ name: 1.0 })) return goal_id, updating
async def create_endpoint(self, endpoint: str, traffic_dict: Dict[str, float], route, methods: List[str]) -> UUID: """Create a new endpoint with the specified route and methods. If the route is None, this is a "headless" endpoint that will not be exposed over HTTP and can only be accessed via a handle. """ async with self.write_lock: self._validate_traffic_dict(traffic_dict) logger.info( "Registering route '{}' to endpoint '{}' with methods '{}'.". format(route, endpoint, methods)) self.endpoint_state.create_endpoint(endpoint, route, methods, TrafficPolicy(traffic_dict))
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str], route_prefix: Optional[str]) -> Optional[GoalId]: if route_prefix is None: route_prefix = f"/{name}" if replica_config.is_asgi_app: # When the backend is asgi application, we want to proxy it # with a prefixed path as well as proxy all HTTP methods. # {wildcard:path} is used so HTTPProxy's Starlette router can match # arbitrary path. if route_prefix.endswith("/"): route_prefix = route_prefix[:-1] http_route = route_prefix + WILDCARD_PATH_SUFFIX http_methods = ALL_HTTP_METHODS else: http_route = route_prefix # Generic endpoint should support a limited subset of HTTP methods. http_methods = ["GET", "POST"] python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: backend_info = BackendInfo(worker_class=create_backend_replica( replica_config.backend_def), version=version, backend_config=backend_config, replica_config=replica_config) goal_id = self.backend_state.deploy_backend(name, backend_info) self.endpoint_state.update_endpoint(name, http_route, http_methods, TrafficPolicy({name: 1.0}), python_methods=python_methods) return goal_id
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str]) -> Optional[GoalId]: """TODO.""" async with self.write_lock: if version is None: version = RESERVED_VERSION_TAG else: if version == RESERVED_VERSION_TAG: # TODO(edoakes): this is unlikely to ever be hit, but it's # still ugly and should be removed once the old codepath # can be deleted. raise ValueError( f"Version {RESERVED_VERSION_TAG} is reserved and " "cannot be used by applications.") goal_id = self.backend_state.deploy_backend( name, backend_config, replica_config, version) self.endpoint_state.create_endpoint(name, f"/{name}", ["GET", "POST"], TrafficPolicy({name: 1.0})) return goal_id
async def deploy(self, name: str, backend_config: BackendConfig, replica_config: ReplicaConfig, version: Optional[str], route_prefix: Optional[str]) -> Optional[GoalId]: if route_prefix is not None: assert route_prefix.startswith("/") if replica_config.is_asgi_app: # When the backend is asgi application, we want to proxy it # with a prefixed path as well as proxy all HTTP methods. http_methods = ALL_HTTP_METHODS else: # Generic endpoint should support a limited subset of HTTP methods. http_methods = ["GET", "POST"] python_methods = [] if inspect.isclass(replica_config.backend_def): for method_name, _ in inspect.getmembers( replica_config.backend_def, inspect.isfunction): python_methods.append(method_name) async with self.write_lock: backend_info = BackendInfo( worker_class=create_backend_replica( replica_config.backend_def), version=version, backend_config=backend_config, replica_config=replica_config) goal_id = self.backend_state.deploy_backend(name, backend_info) endpoint_info = EndpointInfo( http_methods, route=route_prefix, python_methods=python_methods) self.endpoint_state.update_endpoint(name, endpoint_info, TrafficPolicy({ name: 1.0 })) return goal_id