def test_deploy_from_yaml(self, serve_instance):
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "two_deployments.yaml"
)
# Check if yaml string and yaml file both produce the same Application
with open(config_file_name, "r") as f:
app1 = Application.from_yaml(f)
with open(config_file_name, "r") as f:
yaml_str = f.read()
app2 = Application.from_yaml(yaml_str)
compare_specified_options(app1.to_dict(), app2.to_dict())
# Check that deployment works
app1.deploy()
assert (
requests.get("http://localhost:8000/shallow").text == "Hello shallow world!"
)
assert requests.get("http://localhost:8000/one").text == "2"
# Check if yaml string output is same as the Application
recreated_app = Application.from_yaml(app1.to_yaml())
compare_specified_options(recreated_app.to_dict(), app1.to_dict())
# Check if yaml file output is same as the Application
with tempfile.TemporaryFile(mode="w+") as tmp:
app1.to_yaml(tmp)
tmp.seek(0)
compare_specified_options(
Application.from_yaml(tmp).to_dict(), app1.to_dict()
)
def test_immutable_deployment_list(serve_instance):
app = Application([DecoratedClass, decorated_func])
assert len(app.deployments.values()) == 2
for name in app.deployments.keys():
with pytest.raises(RuntimeError):
app.deployments[name] = app.deployments[name].options(
name="sneaky")
def test_add_deployment_valid(self):
app = Application()
app.add_deployment(self.f)
app.add_deployment(self.C)
assert len(app) == 2
assert "f" in app
assert "C" in app
async def get_all_deployments(self, req: Request) -> Response:
from ray.serve.api import list_deployments
from ray.serve.application import Application
app = Application(list(list_deployments().values()))
return Response(
text=json.dumps(app.to_dict()),
content_type="application/json",
)
def test_convert_to_import_path(self, serve_instance):
f = decorated_func.options(name="f")
C = DecoratedClass.options(name="C")
app = Application([f, C])
reconstructed_app = Application.from_yaml(app.to_yaml())
serve.run(reconstructed_app)
assert requests.get(
"http://localhost:8000/f").text == "got decorated func"
assert requests.get(
"http://localhost:8000/C").text == "got decorated class"
def deploy_and_check_responses(
self, deployments, responses, blocking=True, client=None
):
"""
Helper function that deploys the list of deployments, calls them with
their handles, and checks whether they return the objects in responses.
If blocking is False, this function uses a non-blocking deploy and uses
the client to wait until the deployments finish deploying.
"""
Application(deployments).deploy(blocking=blocking)
def check_all_deployed():
try:
for deployment, response in zip(deployments, responses):
if ray.get(deployment.get_handle().remote()) != response:
return False
except Exception:
return False
return True
if blocking:
# If blocking, this should be guaranteed to pass immediately.
assert check_all_deployed()
else:
# If non-blocking, this should pass eventually.
wait_for_condition(check_all_deployed)
def test_valid_deployments(self):
app = Application([self.f, self.C])
assert len(app.deployments) == 2
app_deployment_names = {d.name for d in app.deployments.values()}
assert "f" in app_deployment_names
assert "C" in app_deployment_names
def maybe_build(
node: DeploymentNode, use_build: bool
) -> Union[Application, DeploymentNode]:
if use_build:
return Application.from_dict(build_app(node).to_dict())
else:
return node
def build(target: Union[ClassNode, FunctionNode]) -> Application:
"""Builds a Serve application into a static application.
Takes in a ClassNode or FunctionNode and converts it to a
Serve application consisting of one or more deployments. This is intended
to be used for production scenarios and deployed via the Serve REST API or
CLI, so there are some restrictions placed on the deployments:
1) All of the deployments must be importable. That is, they cannot be
defined in __main__ or inline defined. The deployments will be
imported in production using the same import path they were here.
2) All arguments bound to the deployment must be JSON-serializable.
The returned Application object can be exported to a dictionary or YAML
config.
"""
if in_interactive_shell():
raise RuntimeError(
"build cannot be called from an interactive shell like "
"IPython or Jupyter because it requires all deployments to be "
"importable to run the app after building.")
# TODO(edoakes): this should accept host and port, but we don't
# currently support them in the REST API.
return Application(pipeline_build(target))
async def put_all_deployments(self, req: Request) -> Response:
from ray import serve
from ray.serve.application import Application
app = Application.from_dict(await req.json())
serve.run(app, _blocking=False)
return Response()
def test_invalid_input(self, serve_instance):
"""
Checks Application's deploy behavior when deployment group contains
non-Deployment objects.
"""
with pytest.raises(TypeError):
Application([self.f, self.C,
"not a Deployment object"]).deploy(blocking=True)
def test_add_deployment_repeat_name(self):
with pytest.raises(ValueError):
app = Application()
app.add_deployment(self.f)
app.add_deployment(self.C.options(name="f"))
with pytest.raises(ValueError):
Application([self.C, self.f.options(name="C")])
def test_run_get_ingress_app(serve_instance):
"""Check that serve.run() with an app returns the ingress."""
@serve.deployment(route_prefix=None)
def f():
return "got f"
@serve.deployment(route_prefix="/g")
def g():
return "got g"
app = Application([f, g])
ingress_handle = serve.run(app)
assert ray.get(ingress_handle.remote()) == "got g"
serve_instance.delete_deployments(["f", "g"])
no_ingress_app = Application([f.options(route_prefix="/f"), g])
ingress_handle = serve.run(no_ingress_app)
assert ingress_handle is None
def test_get_set_item(serve_instance):
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "two_deployments.yaml"
)
with open(config_file_name, "r") as f:
app = Application.from_yaml(f)
app["shallow"].deploy()
app["one"].deploy()
assert requests.get("http://localhost:8000/shallow").text == "Hello shallow world!"
assert requests.get("http://localhost:8000/one").text == "2"
async def put_all_deployments(self, req: Request) -> Response:
app = Application.from_dict(await req.json())
app.deploy(blocking=False)
new_names = set()
for deployment in app:
new_names.add(deployment.name)
all_deployments = serve.list_deployments()
all_names = set(all_deployments.keys())
names_to_delete = all_names.difference(new_names)
for name in names_to_delete:
all_deployments[name].delete()
return Response()
def run(
config_or_import_path: str,
runtime_env: str,
runtime_env_json: str,
working_dir: str,
app_dir: str,
address: str,
host: str,
port: int,
blocking: bool,
):
sys.path.insert(0, app_dir)
final_runtime_env = parse_runtime_env_args(
runtime_env=runtime_env,
runtime_env_json=runtime_env_json,
working_dir=working_dir,
)
app_or_node = None
if pathlib.Path(config_or_import_path).is_file():
config_path = config_or_import_path
cli_logger.print(f"Deploying from config file: '{config_path}'.")
with open(config_path, "r") as config_file:
app_or_node = Application.from_yaml(config_file)
else:
import_path = config_or_import_path
cli_logger.print(f"Deploying from import path: '{import_path}'.")
app_or_node = import_attr(import_path)
# Setting the runtime_env here will set defaults for the deployments.
ray.init(address=address, namespace="serve", runtime_env=final_runtime_env)
try:
serve.run(app_or_node, host=host, port=port)
cli_logger.success("Deployed successfully.")
if blocking:
while True:
# Block, letting Ray print logs to the terminal.
time.sleep(10)
except KeyboardInterrupt:
cli_logger.info("Got KeyboardInterrupt, shutting down...")
serve.shutdown()
sys.exit()
async def put_all_deployments(self, req: Request) -> Response:
from ray import serve
from ray.serve.context import get_global_client
from ray.serve.schema import ServeApplicationSchema
from ray.serve.application import Application
config = ServeApplicationSchema.parse_obj(await req.json())
if config.import_path is not None:
client = get_global_client(_override_controller_namespace="serve")
client.deploy_app(config)
else:
# TODO (shrekris-anyscale): Remove this conditional path
app = Application.from_dict(await req.json())
serve.run(app, _blocking=False)
return Response()
def test_deploy_from_dict(self, serve_instance):
config_file_name = os.path.join(os.path.dirname(__file__),
"test_config_files",
"two_deployments.yaml")
with open(config_file_name, "r") as config_file:
config_dict = yaml.safe_load(config_file)
app = Application.from_dict(config_dict)
app_dict = app.to_dict()
compare_specified_options(config_dict, app_dict)
serve.run(app.from_dict(app_dict))
assert (requests.get("http://localhost:8000/shallow").text ==
"Hello shallow world!")
assert requests.get("http://localhost:8000/one").text == "2"
async def put_all_deployments(self, req: Request) -> Response:
from ray import serve
from ray.serve.context import get_global_client
from ray.serve.application import Application
app = Application.from_dict(await req.json())
serve.run(app, _blocking=False)
new_names = set()
for deployment in app.deployments.values():
new_names.add(deployment.name)
all_deployments = serve.list_deployments()
all_names = set(all_deployments.keys())
names_to_delete = all_names.difference(new_names)
get_global_client().delete_deployments(names_to_delete)
return Response()
def test_immutable_deployment_list(serve_instance):
config_file_name = os.path.join(os.path.dirname(__file__),
"test_config_files",
"two_deployments.yaml")
with open(config_file_name, "r") as f:
app = Application.from_yaml(f)
assert len(app.deployments.values()) == 2
for name in app.deployments.keys():
with pytest.raises(RuntimeError):
app.deployments[name] = app.deployments[name].options(
name="sneaky")
for deployment in app.deployments.values():
deployment.deploy()
assert requests.get(
"http://localhost:8000/shallow").text == "Hello shallow world!"
assert requests.get("http://localhost:8000/one").text == "2"
def test_mutual_handles(self, serve_instance):
"""
Atomically deploys a group of deployments that get handles to other
deployments in the group inside their __init__ functions. The handle
references should fail in a non-atomic deployment. Checks whether the
deployments deploy correctly.
"""
@serve.deployment
class MutualHandles:
async def __init__(self, handle_name):
self.handle = serve.get_deployment(handle_name).get_handle()
async def __call__(self, echo: str):
return await self.handle.request_echo.remote(echo)
async def request_echo(self, echo: str):
return echo
names = []
for i in range(10):
names.append("a" * i)
deployments = []
for idx in range(len(names)):
# Each deployment will hold a ServeHandle with the next name in
# the list
deployment_name = names[idx]
handle_name = names[(idx + 1) % len(names)]
deployments.append(
MutualHandles.options(name=deployment_name, init_args=(handle_name,))
)
Application(deployments).deploy(blocking=True)
for deployment in deployments:
assert (ray.get(deployment.get_handle().remote("hello"))) == "hello"
# Check idempotence
for _ in range(2):
subprocess.check_output(["serve", "deploy", config_file_name])
wait_for_condition(lambda: get_num_deployments() == 2, timeout=35)
subprocess.check_output(["serve", "delete", "-y"])
wait_for_condition(lambda: get_num_deployments() == 0, timeout=35)
@serve.deployment
def parrot(request):
return request.query_params["sound"]
parrot_app = Application([parrot])
@pytest.mark.skipif(sys.platform == "win32",
reason="File path incorrect on Windows.")
def test_run_application(ray_start_stop):
# Deploys valid config file and import path via serve run
# Deploy via config file
config_file_name = os.path.join(os.path.dirname(__file__),
"test_config_files",
"two_deployments.yaml")
p = subprocess.Popen(["serve", "run", "--address=auto", config_file_name])
wait_for_condition(lambda: ping_endpoint("one") == "2", timeout=10)
wait_for_condition(
def run(
config_or_import_path: str,
args_and_kwargs: Tuple[str],
runtime_env: str,
runtime_env_json: str,
working_dir: str,
address: str,
):
# Check if path provided is for config or import
is_config = pathlib.Path(config_or_import_path).is_file()
args, kwargs = _process_args_and_kwargs(args_and_kwargs)
# Calculate deployments' runtime env updates requested via args
runtime_env_updates = parse_runtime_env_args(
runtime_env=runtime_env,
runtime_env_json=runtime_env_json,
working_dir=working_dir,
)
# Create ray.init()'s runtime_env
if "working_dir" in runtime_env_updates:
ray_runtime_env = {
"working_dir": runtime_env_updates.pop("working_dir")
}
else:
ray_runtime_env = {}
if is_config:
config_path = config_or_import_path
# Delay serve.start() to catch invalid inputs without waiting
if len(args) + len(kwargs) > 0:
raise ValueError(
"ARGS_AND_KWARGS cannot be defined for a "
"config file deployment. Please specify the "
"init_args and init_kwargs inside the config file.")
cli_logger.print("Deploying application in config file at "
f"{config_path}.")
with open(config_path, "r") as config_file:
app = Application.from_yaml(config_file)
else:
import_path = config_or_import_path
if "." not in import_path:
raise ValueError(
"Import paths must be of the form "
'"module.submodule_1...submodule_n.MyClassOrFunction".')
cli_logger.print(
f'Deploying function or class imported from "{import_path}".')
deployment_name = import_path[import_path.rfind(".") + 1:]
deployment = serve.deployment(name=deployment_name)(import_path)
app = Application(
[deployment.options(init_args=args, init_kwargs=kwargs)])
ray.init(address=address, namespace="serve", runtime_env=ray_runtime_env)
for deployment in app:
_configure_runtime_env(deployment, runtime_env_updates)
app.run(logger=cli_logger)
def test_repeated_deployment_names(self):
with pytest.raises(ValueError):
Application([self.f, self.C.options(name="f")])
with pytest.raises(ValueError):
Application([self.C, self.f.options(name="C")])
def test_non_deployments(self):
with pytest.raises(TypeError):
Application([self.f, 5, "hello"])
