def create_backend(func_or_class, backend_tag, *actor_init_args):
"""Create a backend using func_or_class and assign backend_tag.
Args:
func_or_class (callable, class): a function or a class implements
__call__ protocol.
backend_tag (str): a unique tag assign to this backend. It will be used
to associate services in traffic policy.
*actor_init_args (optional): the argument to pass to the class
initialization method.
"""
if inspect.isfunction(func_or_class):
# ignore lint on lambda expression
creator = lambda: TaskRunnerActor.remote(func_or_class) # noqa: E731
elif inspect.isclass(func_or_class):
# Python inheritance order is right-to-left. We put RayServeMixin
# on the left to make sure its methods are not overriden.
@ray.remote
class CustomActor(RayServeMixin, func_or_class):
pass
# ignore lint on lambda expression
creator = lambda: CustomActor.remote(*actor_init_args) # noqa: E731
else:
raise TypeError(
"Backend must be a function or class, it is {}.".format(
type(func_or_class)))
global_state.backend_table.register_backend(backend_tag, creator)
scale(backend_tag, 1)
def test_task_runner_check_context(serve_instance):
q = CentralizedQueuesActor.remote()
def echo(flask_request, i=None):
# Accessing the flask_request without web context should throw.
return flask_request.args["i"]
CONSUMER_NAME = "runner"
PRODUCER_NAME = "producer"
runner = TaskRunnerActor.remote(echo)
runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner)
runner._ray_serve_main_loop.remote()
q.link.remote(PRODUCER_NAME, CONSUMER_NAME)
result_token = ray.ObjectID(
ray.get(
q.enqueue_request.remote(
PRODUCER_NAME,
request_args=None,
request_kwargs={"i": 42},
request_context=context.TaskContext.Python)))
with pytest.raises(ray.exceptions.RayTaskError):
ray.get(result_token)
def create_backend(func_or_class, backend_tag, *actor_init_args):
"""Create a backend using func_or_class and assign backend_tag.
Args:
func_or_class (callable, class): a function or a class implements
__call__ protocol.
backend_tag (str): a unique tag assign to this backend. It will be used
to associate services in traffic policy.
*actor_init_args (optional): the argument to pass to the class
initialization method.
"""
if inspect.isfunction(func_or_class):
runner = TaskRunnerActor.remote(func_or_class)
elif inspect.isclass(func_or_class):
# Python inheritance order is right-to-left. We put RayServeMixin
# on the left to make sure its methods are not overriden.
@ray.remote
class CustomActor(RayServeMixin, func_or_class):
pass
runner = CustomActor.remote(*actor_init_args)
else:
raise TypeError(
"Backend must be a function or class, it is {}.".format(
type(func_or_class)))
global_state.backend_actor_handles.append(runner)
runner._ray_serve_setup.remote(backend_tag,
global_state.router_actor_handle)
runner._ray_serve_main_loop.remote(runner)
global_state.registered_backends.add(backend_tag)
def test_runner_actor(serve_instance):
q = CentralizedQueuesActor.remote()
def echo(flask_request, i=None):
return i
CONSUMER_NAME = "runner"
PRODUCER_NAME = "prod"
runner = TaskRunnerActor.remote(echo)
runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner)
runner._ray_serve_main_loop.remote()
q.link.remote(PRODUCER_NAME, CONSUMER_NAME)
for query in [333, 444, 555]:
result_token = ray.ObjectID(
ray.get(
q.enqueue_request.remote(
PRODUCER_NAME,
request_args=None,
request_kwargs={"i": query},
request_context=context.TaskContext.Python)))
assert ray.get(result_token) == query
def test_runner_actor(serve_instance):
q = CentralizedQueuesActor.remote()
def echo(i):
return i
CONSUMER_NAME = "runner"
PRODUCER_NAME = "prod"
runner = TaskRunnerActor.remote(echo)
runner._ray_serve_setup.remote(CONSUMER_NAME, q)
runner._ray_serve_main_loop.remote(runner)
q.link.remote(PRODUCER_NAME, CONSUMER_NAME)
for query in [333, 444, 555]:
result_token = ray.ObjectID(
ray.get(q.enqueue_request.remote(PRODUCER_NAME, query)))
assert ray.get(result_token) == query
async def test_runner_actor(serve_instance):
q = RoundRobinPolicyQueueActor.remote()
def echo(flask_request, i=None):
return i
CONSUMER_NAME = "runner"
PRODUCER_NAME = "prod"
runner = TaskRunnerActor.remote(echo)
runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner)
runner._ray_serve_fetch.remote()
q.link.remote(PRODUCER_NAME, CONSUMER_NAME)
for query in [333, 444, 555]:
query_param = RequestMetadata(PRODUCER_NAME,
context.TaskContext.Python)
result = await q.enqueue_request.remote(query_param, i=query)
assert result == query
async def test_task_runner_check_context(serve_instance):
q = RoundRobinPolicyQueueActor.remote()
def echo(flask_request, i=None):
# Accessing the flask_request without web context should throw.
return flask_request.args["i"]
CONSUMER_NAME = "runner"
PRODUCER_NAME = "producer"
runner = TaskRunnerActor.remote(echo)
runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner)
runner._ray_serve_fetch.remote()
q.link.remote(PRODUCER_NAME, CONSUMER_NAME)
query_param = RequestMetadata(PRODUCER_NAME, context.TaskContext.Python)
result_oid = q.enqueue_request.remote(query_param, i=42)
with pytest.raises(ray.exceptions.RayTaskError):
await result_oid
def create_backend(func_or_class,
backend_tag,
*actor_init_args,
backend_config=BackendConfig()):
"""Create a backend using func_or_class and assign backend_tag.
Args:
func_or_class (callable, class): a function or a class implements
__call__ protocol.
backend_tag (str): a unique tag assign to this backend. It will be used
to associate services in traffic policy.
backend_config (BackendConfig): An object defining backend properties
for starting a backend.
*actor_init_args (optional): the argument to pass to the class
initialization method.
"""
assert isinstance(backend_config,
BackendConfig), ("backend_config must be"
" of instance BackendConfig")
backend_config_dict = dict(backend_config)
should_accept_batch = (True if backend_config.max_batch_size is not None
else False)
batch_annotation_not_found = RayServeException(
"max_batch_size is set in config but the function or method does not "
"accept batching. Please use @serve.accept_batch to explicitly mark "
"the function or method as batchable and takes in list as arguments.")
arg_list = []
if inspect.isfunction(func_or_class):
if should_accept_batch and not hasattr(func_or_class,
"serve_accept_batch"):
raise batch_annotation_not_found
# arg list for a fn is function itself
arg_list = [func_or_class]
# ignore lint on lambda expression
creator = lambda kwrgs: TaskRunnerActor._remote(**kwrgs) # noqa: E731
elif inspect.isclass(func_or_class):
if should_accept_batch and not hasattr(func_or_class.__call__,
"serve_accept_batch"):
raise batch_annotation_not_found
# Python inheritance order is right-to-left. We put RayServeMixin
# on the left to make sure its methods are not overriden.
@ray.remote
class CustomActor(RayServeMixin, func_or_class):
pass
arg_list = actor_init_args
# ignore lint on lambda expression
creator = lambda kwargs: CustomActor._remote(**kwargs) # noqa: E731
else:
raise TypeError(
"Backend must be a function or class, it is {}.".format(
type(func_or_class)))
# save creator which starts replicas
global_state.backend_table.register_backend(backend_tag, creator)
# save information about configurations needed to start the replicas
global_state.backend_table.register_info(backend_tag, backend_config_dict)
# save the initial arguments needed by replicas
global_state.backend_table.save_init_args(backend_tag, arg_list)
# set the backend config inside the router
# particularly for max-batch-size
ray.get(global_state.init_or_get_router().set_backend_config.remote(
backend_tag, backend_config_dict))
scale(backend_tag, backend_config_dict["num_replicas"])