async def __init__(self, controller_name: str, http_config: HTTPOptions, detached: bool = False): # Used to read/write checkpoints. self.kv_store = RayInternalKVStore(namespace=controller_name) # Dictionary of backend_tag -> proxy_name -> most recent queue length. self.backend_stats = defaultdict(lambda: defaultdict(dict)) # Used to ensure that only a single state-changing operation happens # at any given time. self.write_lock = asyncio.Lock() # Map of awaiting results # TODO(ilr): Checkpoint this once this becomes asynchronous self.inflight_results: Dict[UUID, asyncio.Event] = dict() self._serializable_inflight_results: Dict[UUID, FutureResult] = dict() # HTTP state doesn't currently require a checkpoint. self.http_state = HTTPState(controller_name, detached, http_config) checkpoint_bytes = self.kv_store.get(CHECKPOINT_KEY) if checkpoint_bytes is None: logger.debug("No checkpoint found") self.backend_state = BackendState(controller_name, detached) self.endpoint_state = EndpointState() else: checkpoint: Checkpoint = pickle.loads(checkpoint_bytes) self.backend_state = BackendState( controller_name, detached, checkpoint=checkpoint.backend_state_checkpoint) self.endpoint_state = EndpointState( checkpoint=checkpoint.endpoint_state_checkpoint) self._serializable_inflight_results = checkpoint.inflight_reqs for uuid, fut_result in self._serializable_inflight_results.items( ): self._create_event_with_result(fut_result.requested_goal, uuid) # NOTE(simon): Currently we do all-to-all broadcast. This means # any listeners will receive notification for all changes. This # can be problem at scale, e.g. updating a single backend config # will send over the entire configs. In the future, we should # optimize the logic to support subscription by key. self.long_poll_host = LongPollHost() self.notify_backend_configs_changed() self.notify_replica_handles_changed() self.notify_traffic_policies_changed() self.notify_route_table_changed() asyncio.get_event_loop().create_task(self.run_control_loop())
async def __init__(self, controller_name: str, http_config: HTTPOptions, detached: bool = False): # Used to read/write checkpoints. self.kv_store = RayInternalKVStore(namespace=controller_name) # Dictionary of backend_tag -> proxy_name -> most recent queue length. self.backend_stats = defaultdict(lambda: defaultdict(dict)) # Used to ensure that only a single state-changing operation happens # at any given time. self.write_lock = asyncio.Lock() # NOTE(simon): Currently we do all-to-all broadcast. This means # any listeners will receive notification for all changes. This # can be problem at scale, e.g. updating a single backend config # will send over the entire configs. In the future, we should # optimize the logic to support subscription by key. self.long_poll_host = LongPollHost() self.goal_manager = AsyncGoalManager() self.http_state = HTTPState(controller_name, detached, http_config) self.endpoint_state = EndpointState(self.kv_store, self.long_poll_host) self.backend_state = BackendState(controller_name, detached, self.kv_store, self.long_poll_host, self.goal_manager) asyncio.get_event_loop().create_task(self.run_control_loop())
def mock_backend_state_inputs() -> Tuple[BackendState, Mock, Mock]: with patch( "ray.serve.kv_store.RayInternalKVStore") as mock_kv_store, patch( "ray.serve.long_poll.LongPollHost") as mock_long_poll: mock_kv_store.get = Mock(return_value=None) backend_state = BackendState("name", True, mock_kv_store, mock_long_poll) yield backend_state, mock_kv_store, mock_long_poll
def mock_backend_state() -> Tuple[BackendState, Mock, Mock]: timer = MockTimer() with patch( "ray.serve.backend_state.ActorReplicaWrapper", new=MockReplicaActorWrapper), patch( "time.time", new=timer.time), patch("ray.serve.kv_store.RayInternalKVStore" ) as mock_kv_store, patch( "ray.serve.long_poll.LongPollHost" ) as mock_long_poll, patch.object( BackendState, "_checkpoint") as mock_checkpoint: mock_kv_store.get = Mock(return_value=None) goal_manager = AsyncGoalManager() backend_state = BackendState("name", True, mock_kv_store, mock_long_poll, goal_manager) mock_checkpoint.return_value = None yield backend_state, timer, goal_manager
async def __init__(self, controller_name: str, http_config: HTTPOptions, detached: bool = False): # Used to read/write checkpoints. self.kv_store = RayInternalKVStore(namespace=controller_name) # Dictionary of backend_tag -> proxy_name -> most recent queue length. self.backend_stats = defaultdict(lambda: defaultdict(dict)) # Used to ensure that only a single state-changing operation happens # at any given time. self.write_lock = asyncio.Lock() self.long_poll_host = LongPollHost() self.goal_manager = AsyncGoalManager() self.http_state = HTTPState(controller_name, detached, http_config) self.endpoint_state = EndpointState(self.kv_store, self.long_poll_host) self.backend_state = BackendState(controller_name, detached, self.kv_store, self.long_poll_host, self.goal_manager) asyncio.get_event_loop().create_task(self.run_control_loop())