def test_options_and_names(serve_instance):
m1 = Adder.bind(1)
m1_built = pipeline_build(m1)[-1]
assert m1_built.name == "Adder"
m1 = Adder.options(name="Adder2").bind(1)
m1_built = pipeline_build(m1)[-1]
assert m1_built.name == "Adder2"
m1 = Adder.options(num_replicas=2).bind(1)
m1_built = pipeline_build(m1)[-1]
assert m1_built.num_replicas == 2
def test_deploment_options_func_class_with_class_method():
with InputNode() as dag_input:
counter = Counter.bind()
m1 = Model.options(name="m1", max_concurrent_queries=3).bind(1)
m2 = Model.options(name="m2", max_concurrent_queries=5).bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.options(num_replicas=3, max_concurrent_queries=7).bind(
m1_output, m2_output, kwargs_output=dag_input[2]
)
dag = counter.__call__.bind(combine_output)
serve_dag = Driver.bind(dag)
deployments = pipeline_build(serve_dag)
hit_count = 0
for deployment in deployments:
if deployment.name == "counter":
assert deployment.num_replicas == 2
assert deployment.user_config == {"count": 123, "b": 2}
hit_count += 1
elif deployment.name == "m1":
assert deployment.max_concurrent_queries == 3
hit_count += 1
elif deployment.name == "m2":
assert deployment.max_concurrent_queries == 5
hit_count += 1
elif deployment.name == "combine":
assert deployment.num_replicas == 3
assert deployment.max_concurrent_queries == 7
hit_count += 1
assert hit_count == 4, "Not all deployments with expected name were found."
def build(target: Union[ClassNode, FunctionNode]) -> Application:
"""Builds a Serve application into a static application.
Takes in a ClassNode or FunctionNode and converts it to a
Serve application consisting of one or more deployments. This is intended
to be used for production scenarios and deployed via the Serve REST API or
CLI, so there are some restrictions placed on the deployments:
1) All of the deployments must be importable. That is, they cannot be
defined in __main__ or inline defined. The deployments will be
imported in production using the same import path they were here.
2) All arguments bound to the deployment must be JSON-serializable.
The returned Application object can be exported to a dictionary or YAML
config.
"""
if in_interactive_shell():
raise RuntimeError(
"build cannot be called from an interactive shell like "
"IPython or Jupyter because it requires all deployments to be "
"importable to run the app after building."
)
# TODO(edoakes): this should accept host and port, but we don't
# currently support them in the REST API.
return Application(pipeline_build(target))
def test_nested_building():
with InputNode() as inp:
out = func.bind(inp)
out = Driver.bind().__call__.bind(out)
out = func.bind(out)
dag = Driver.bind(out, func.bind())
assert len(pipeline_build(dag)) == 5
def test_http_we_provide_default_route_prefix_func(serve_instance):
"""Ensure the default ingress deployment route is '/' instead of driver
function name
"""
func_dag = combine.bind(1, 2)
deployments = pipeline_build(func_dag)
ingress_deployment = get_and_validate_ingress_deployment(deployments)
assert ingress_deployment.route_prefix == "/"
def test_http_reconfigure_non_default_route_prefix_on_root(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
non_root_deployment = deployments[-1].options(route_prefix="/yoo")
deployments[-1] = non_root_deployment
_ = get_and_validate_ingress_deployment(deployments)
def test_http_no_non_ingress_deployment_rout_prefix(serve_instance):
with InputNode() as dag_input:
m1 = Model.options(route_prefix="/should-fail").bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
with pytest.raises(
ValueError,
match="Route prefix is only configurable on the ingress deployment",
):
_ = pipeline_build(serve_dag)
def test_http_user_bring_own_driver_route_prefix(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.options(route_prefix="/hello").bind(combine_output)
deployments = pipeline_build(serve_dag)
ingress_deployment = get_and_validate_ingress_deployment(deployments)
assert ingress_deployment.route_prefix == "/hello"
for deployment in deployments[:-1]:
assert deployment.route_prefix is None
def test_http_we_provide_default_route_prefix_cls(serve_instance):
"""Ensure the default ingress deployment route is '/' instead of driver
class name
"""
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
ingress_deployment = get_and_validate_ingress_deployment(deployments)
assert ingress_deployment.route_prefix == "/"
for deployment in deployments[:-1]:
assert deployment.route_prefix is None
def test_http_only_one_ingress_deployment(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
non_root_deployment = deployments[0].options(route_prefix="/")
deployments[0] = non_root_deployment
with pytest.raises(
ValueError,
match=(
"Only one deployment in an Serve Application or DAG can have "
"non-None route prefix"
),
):
_ = get_and_validate_ingress_deployment(deployments)
def run(
target: Union[ClassNode, FunctionNode],
_blocking: bool = True,
*,
host: str = DEFAULT_HTTP_HOST,
port: int = DEFAULT_HTTP_PORT,
) -> Optional[RayServeHandle]:
"""Run a Serve application and return a ServeHandle to the ingress.
Either a ClassNode, FunctionNode, or a pre-built application
can be passed in. If a node is passed in, all of the deployments it depends
on will be deployed. If there is an ingress, its handle will be returned.
Args:
target (Union[ClassNode, FunctionNode, Application]):
A user-built Serve Application or a ClassNode that acts as the
root node of DAG. By default ClassNode is the Driver
deployment unless user provides a customized one.
host (str): The host passed into serve.start().
port (int): The port passed into serve.start().
Returns:
RayServeHandle: A regular ray serve handle that can be called by user
to execute the serve DAG.
"""
client = start(detached=True, http_options={"host": host, "port": port})
if isinstance(target, Application):
deployments = list(target.deployments.values())
ingress = target.ingress
# Each DAG should always provide a valid Driver ClassNode
elif isinstance(target, ClassNode):
deployments = pipeline_build(target)
ingress = get_and_validate_ingress_deployment(deployments)
# Special case where user is doing single function serve.run(func.bind())
elif isinstance(target, FunctionNode):
deployments = pipeline_build(target)
ingress = get_and_validate_ingress_deployment(deployments)
if len(deployments) != 1:
raise ValueError(
"We only support single function node in serve.run, ex: "
"serve.run(func.bind()). For more than one nodes in your DAG, "
"Please provide a driver class and bind it as entrypoint to "
"your Serve DAG."
)
elif isinstance(target, DAGNode):
raise ValueError(
"Invalid DAGNode type as entry to serve.run(), "
f"type: {type(target)}, accepted: ClassNode, "
"FunctionNode please provide a driver class and bind it "
"as entrypoint to your Serve DAG."
)
else:
raise TypeError(
"Expected a ClassNode, FunctionNode, or Application as target. "
f"Got unexpected type {type(target)} instead."
)
parameter_group = []
for deployment in deployments:
deployment_parameters = {
"name": deployment._name,
"func_or_class": deployment._func_or_class,
"init_args": deployment.init_args,
"init_kwargs": deployment.init_kwargs,
"ray_actor_options": deployment._ray_actor_options,
"config": deployment._config,
"version": deployment._version,
"prev_version": deployment._prev_version,
"route_prefix": deployment.route_prefix,
"url": deployment.url,
}
parameter_group.append(deployment_parameters)
client.deploy_group(
parameter_group, _blocking=_blocking, remove_past_deployments=True
)
if ingress is not None:
return ingress.get_handle()
def test_dag_building():
dag = Driver.bind(func.bind())
assert len(pipeline_build(dag)) == 2
def test_class_building():
dag = Driver.bind()
assert len(pipeline_build(dag)) == 1
def test_func_building():
dag = func.bind()
assert len(pipeline_build(dag)) == 1
