def get_handle(endpoint_name,
relative_slo_ms=None,
absolute_slo_ms=None,
missing_ok=False):
"""Retrieve RayServeHandle for service endpoint to invoke it from Python.
Args:
endpoint_name (str): A registered service endpoint.
relative_slo_ms(float): Specify relative deadline in milliseconds for
queries fired using this handle. (Default: None)
absolute_slo_ms(float): Specify absolute deadline in milliseconds for
queries fired using this handle. (Default: None)
missing_ok (bool): If true, skip the check for the endpoint existence.
It can be useful when the endpoint has not been registered.
Returns:
RayServeHandle
"""
if not missing_ok:
assert endpoint_name in expand(
global_state.route_table.list_service(
include_headless=True).values())
# Delay import due to it's dependency on global_state
from ray.serve.handle import RayServeHandle
return RayServeHandle(
global_state.init_or_get_router(),
endpoint_name,
relative_slo_ms,
absolute_slo_ms,
)
def split(endpoint_name, traffic_policy_dictionary):
"""Associate a service endpoint with traffic policy.
Example:
>>> serve.split("service-name", {
"backend:v1": 0.5,
"backend:v2": 0.5
})
Args:
endpoint_name (str): A registered service endpoint.
traffic_policy_dictionary (dict): a dictionary maps backend names
to their traffic weights. The weights must sum to 1.
"""
assert endpoint_name in expand(
global_state.route_table.list_service(include_headless=True).values())
assert isinstance(traffic_policy_dictionary,
dict), "Traffic policy must be dictionary"
prob = 0
for backend, weight in traffic_policy_dictionary.items():
prob += weight
assert (backend in global_state.backend_table.list_backends()
), "backend {} is not registered".format(backend)
assert np.isclose(
prob, 1,
atol=0.02), "weights must sum to 1, currently it sums to {}".format(
prob)
global_state.policy_table.register_traffic_policy(
endpoint_name, traffic_policy_dictionary)
ray.get(global_state.init_or_get_router().set_traffic.remote(
endpoint_name, traffic_policy_dictionary))
def split_traffic(self, endpoint_name, traffic_policy_dictionary):
assert endpoint_name in expand(
self.route_table.list_service(include_headless=True).values())
assert isinstance(traffic_policy_dictionary,
dict), "Traffic policy must be dictionary"
prob = 0
for backend, weight in traffic_policy_dictionary.items():
prob += weight
assert (backend in self.backend_table.list_backends()
), "backend {} is not registered".format(backend)
assert np.isclose(
prob, 1, atol=0.02
), "weights must sum to 1, currently it sums to {}".format(prob)
self.policy_table.register_traffic_policy(endpoint_name,
traffic_policy_dictionary)
[router] = self.get_router()
ray.get(
router.set_traffic.remote(endpoint_name,
traffic_policy_dictionary))
def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None):
"""Retrieve RayServeHandle for service endpoint to invoke it from Python.
Args:
endpoint_name (str): A registered service endpoint.
relative_slo_ms(float): Specify relative deadline in milliseconds for
queries fired using this handle. (Default: None)
absolute_slo_ms(float): Specify absolute deadline in milliseconds for
queries fired using this handle. (Default: None)
Returns:
RayServeHandle
"""
assert endpoint_name in expand(
global_state.route_table.list_service(include_headless=True).values())
# Delay import due to it's dependency on global_state
from ray.serve.handle import RayServeHandle
return RayServeHandle(global_state.init_or_get_router(), endpoint_name,
relative_slo_ms, absolute_slo_ms)
def get_all_endpoints(self):
return expand(
self.route_table.list_service(include_headless=True).values())
def get_all_routes(self):
return expand(self.route_table.list_service().keys())
