def create_backend(func_or_class, backend_tag, *actor_init_args): """Create a backend using func_or_class and assign backend_tag. Args: func_or_class (callable, class): a function or a class implements __call__ protocol. backend_tag (str): a unique tag assign to this backend. It will be used to associate services in traffic policy. *actor_init_args (optional): the argument to pass to the class initialization method. """ if inspect.isfunction(func_or_class): # ignore lint on lambda expression creator = lambda: TaskRunnerActor.remote(func_or_class) # noqa: E731 elif inspect.isclass(func_or_class): # Python inheritance order is right-to-left. We put RayServeMixin # on the left to make sure its methods are not overriden. @ray.remote class CustomActor(RayServeMixin, func_or_class): pass # ignore lint on lambda expression creator = lambda: CustomActor.remote(*actor_init_args) # noqa: E731 else: raise TypeError( "Backend must be a function or class, it is {}.".format( type(func_or_class))) global_state.backend_table.register_backend(backend_tag, creator) scale(backend_tag, 1)
def test_task_runner_check_context(serve_instance): q = CentralizedQueuesActor.remote() def echo(flask_request, i=None): # Accessing the flask_request without web context should throw. return flask_request.args["i"] CONSUMER_NAME = "runner" PRODUCER_NAME = "producer" runner = TaskRunnerActor.remote(echo) runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner) runner._ray_serve_main_loop.remote() q.link.remote(PRODUCER_NAME, CONSUMER_NAME) result_token = ray.ObjectID( ray.get( q.enqueue_request.remote( PRODUCER_NAME, request_args=None, request_kwargs={"i": 42}, request_context=context.TaskContext.Python))) with pytest.raises(ray.exceptions.RayTaskError): ray.get(result_token)
def create_backend(func_or_class, backend_tag, *actor_init_args): """Create a backend using func_or_class and assign backend_tag. Args: func_or_class (callable, class): a function or a class implements __call__ protocol. backend_tag (str): a unique tag assign to this backend. It will be used to associate services in traffic policy. *actor_init_args (optional): the argument to pass to the class initialization method. """ if inspect.isfunction(func_or_class): runner = TaskRunnerActor.remote(func_or_class) elif inspect.isclass(func_or_class): # Python inheritance order is right-to-left. We put RayServeMixin # on the left to make sure its methods are not overriden. @ray.remote class CustomActor(RayServeMixin, func_or_class): pass runner = CustomActor.remote(*actor_init_args) else: raise TypeError( "Backend must be a function or class, it is {}.".format( type(func_or_class))) global_state.backend_actor_handles.append(runner) runner._ray_serve_setup.remote(backend_tag, global_state.router_actor_handle) runner._ray_serve_main_loop.remote(runner) global_state.registered_backends.add(backend_tag)
def test_runner_actor(serve_instance): q = CentralizedQueuesActor.remote() def echo(flask_request, i=None): return i CONSUMER_NAME = "runner" PRODUCER_NAME = "prod" runner = TaskRunnerActor.remote(echo) runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner) runner._ray_serve_main_loop.remote() q.link.remote(PRODUCER_NAME, CONSUMER_NAME) for query in [333, 444, 555]: result_token = ray.ObjectID( ray.get( q.enqueue_request.remote( PRODUCER_NAME, request_args=None, request_kwargs={"i": query}, request_context=context.TaskContext.Python))) assert ray.get(result_token) == query
def test_runner_actor(serve_instance): q = CentralizedQueuesActor.remote() def echo(i): return i CONSUMER_NAME = "runner" PRODUCER_NAME = "prod" runner = TaskRunnerActor.remote(echo) runner._ray_serve_setup.remote(CONSUMER_NAME, q) runner._ray_serve_main_loop.remote(runner) q.link.remote(PRODUCER_NAME, CONSUMER_NAME) for query in [333, 444, 555]: result_token = ray.ObjectID( ray.get(q.enqueue_request.remote(PRODUCER_NAME, query))) assert ray.get(result_token) == query
async def test_runner_actor(serve_instance): q = RoundRobinPolicyQueueActor.remote() def echo(flask_request, i=None): return i CONSUMER_NAME = "runner" PRODUCER_NAME = "prod" runner = TaskRunnerActor.remote(echo) runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner) runner._ray_serve_fetch.remote() q.link.remote(PRODUCER_NAME, CONSUMER_NAME) for query in [333, 444, 555]: query_param = RequestMetadata(PRODUCER_NAME, context.TaskContext.Python) result = await q.enqueue_request.remote(query_param, i=query) assert result == query
async def test_task_runner_check_context(serve_instance): q = RoundRobinPolicyQueueActor.remote() def echo(flask_request, i=None): # Accessing the flask_request without web context should throw. return flask_request.args["i"] CONSUMER_NAME = "runner" PRODUCER_NAME = "producer" runner = TaskRunnerActor.remote(echo) runner._ray_serve_setup.remote(CONSUMER_NAME, q, runner) runner._ray_serve_fetch.remote() q.link.remote(PRODUCER_NAME, CONSUMER_NAME) query_param = RequestMetadata(PRODUCER_NAME, context.TaskContext.Python) result_oid = q.enqueue_request.remote(query_param, i=42) with pytest.raises(ray.exceptions.RayTaskError): await result_oid
def create_backend(func_or_class, backend_tag, *actor_init_args, backend_config=BackendConfig()): """Create a backend using func_or_class and assign backend_tag. Args: func_or_class (callable, class): a function or a class implements __call__ protocol. backend_tag (str): a unique tag assign to this backend. It will be used to associate services in traffic policy. backend_config (BackendConfig): An object defining backend properties for starting a backend. *actor_init_args (optional): the argument to pass to the class initialization method. """ assert isinstance(backend_config, BackendConfig), ("backend_config must be" " of instance BackendConfig") backend_config_dict = dict(backend_config) should_accept_batch = (True if backend_config.max_batch_size is not None else False) batch_annotation_not_found = RayServeException( "max_batch_size is set in config but the function or method does not " "accept batching. Please use @serve.accept_batch to explicitly mark " "the function or method as batchable and takes in list as arguments.") arg_list = [] if inspect.isfunction(func_or_class): if should_accept_batch and not hasattr(func_or_class, "serve_accept_batch"): raise batch_annotation_not_found # arg list for a fn is function itself arg_list = [func_or_class] # ignore lint on lambda expression creator = lambda kwrgs: TaskRunnerActor._remote(**kwrgs) # noqa: E731 elif inspect.isclass(func_or_class): if should_accept_batch and not hasattr(func_or_class.__call__, "serve_accept_batch"): raise batch_annotation_not_found # Python inheritance order is right-to-left. We put RayServeMixin # on the left to make sure its methods are not overriden. @ray.remote class CustomActor(RayServeMixin, func_or_class): pass arg_list = actor_init_args # ignore lint on lambda expression creator = lambda kwargs: CustomActor._remote(**kwargs) # noqa: E731 else: raise TypeError( "Backend must be a function or class, it is {}.".format( type(func_or_class))) # save creator which starts replicas global_state.backend_table.register_backend(backend_tag, creator) # save information about configurations needed to start the replicas global_state.backend_table.register_info(backend_tag, backend_config_dict) # save the initial arguments needed by replicas global_state.backend_table.save_init_args(backend_tag, arg_list) # set the backend config inside the router # particularly for max-batch-size ray.get(global_state.init_or_get_router().set_backend_config.remote( backend_tag, backend_config_dict)) scale(backend_tag, backend_config_dict["num_replicas"])