def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None, missing_ok=False): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) missing_ok (bool): If true, skip the check for the endpoint existence. It can be useful when the endpoint has not been registered. Returns: RayServeHandle """ if not missing_ok: assert endpoint_name in expand( global_state.route_table.list_service( include_headless=True).values()) # Delay import due to it's dependency on global_state from ray.serve.handle import RayServeHandle return RayServeHandle( global_state.init_or_get_router(), endpoint_name, relative_slo_ms, absolute_slo_ms, )
def split(endpoint_name, traffic_policy_dictionary): """Associate a service endpoint with traffic policy. Example: >>> serve.split("service-name", { "backend:v1": 0.5, "backend:v2": 0.5 }) Args: endpoint_name (str): A registered service endpoint. traffic_policy_dictionary (dict): a dictionary maps backend names to their traffic weights. The weights must sum to 1. """ assert endpoint_name in expand( global_state.route_table.list_service(include_headless=True).values()) assert isinstance(traffic_policy_dictionary, dict), "Traffic policy must be dictionary" prob = 0 for backend, weight in traffic_policy_dictionary.items(): prob += weight assert (backend in global_state.backend_table.list_backends() ), "backend {} is not registered".format(backend) assert np.isclose( prob, 1, atol=0.02), "weights must sum to 1, currently it sums to {}".format( prob) global_state.policy_table.register_traffic_policy( endpoint_name, traffic_policy_dictionary) ray.get(global_state.init_or_get_router().set_traffic.remote( endpoint_name, traffic_policy_dictionary))
def split_traffic(self, endpoint_name, traffic_policy_dictionary): assert endpoint_name in expand( self.route_table.list_service(include_headless=True).values()) assert isinstance(traffic_policy_dictionary, dict), "Traffic policy must be dictionary" prob = 0 for backend, weight in traffic_policy_dictionary.items(): prob += weight assert (backend in self.backend_table.list_backends() ), "backend {} is not registered".format(backend) assert np.isclose( prob, 1, atol=0.02 ), "weights must sum to 1, currently it sums to {}".format(prob) self.policy_table.register_traffic_policy(endpoint_name, traffic_policy_dictionary) [router] = self.get_router() ray.get( router.set_traffic.remote(endpoint_name, traffic_policy_dictionary))
def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) Returns: RayServeHandle """ assert endpoint_name in expand( global_state.route_table.list_service(include_headless=True).values()) # Delay import due to it's dependency on global_state from ray.serve.handle import RayServeHandle return RayServeHandle(global_state.init_or_get_router(), endpoint_name, relative_slo_ms, absolute_slo_ms)
def get_all_endpoints(self): return expand( self.route_table.list_service(include_headless=True).values())
def get_all_routes(self): return expand(self.route_table.list_service().keys())