def test_deploy_app_update_config(self, client: ServeControllerClient):
config = ServeApplicationSchema.parse_obj(self.get_test_config())
client.deploy_app(config)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "4 pizzas please!"
)
config = self.get_test_config()
config["deployments"] = [
{
"name": "Adder",
"user_config": {
"increment": -1,
},
},
]
client.deploy_app(ServeApplicationSchema.parse_obj(config))
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "1 pizzas please!"
)
def test_deploy_app_overwrite_apps(self, client: ServeControllerClient):
"""Check that overwriting a live app with a new one works."""
# Launch first graph. Its driver's route_prefix should be "/".
test_config_1 = ServeApplicationSchema.parse_obj(
{
"import_path": "ray.serve.tests.test_config_files.world.DagNode",
}
)
client.deploy_app(test_config_1)
wait_for_condition(
lambda: requests.get("http://localhost:8000/").text == "wonderful world"
)
# Launch second graph. Its driver's route_prefix should also be "/".
# "/" should lead to the new driver.
test_config_2 = ServeApplicationSchema.parse_obj(
{
"import_path": "ray.serve.tests.test_config_files.pizza.serve_dag",
}
)
client.deploy_app(test_config_2)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "4 pizzas please!"
)
def test_deploy_app_update_timestamp(self, client: ServeControllerClient):
assert client.get_serve_status().app_status.deployment_timestamp == 0
config = ServeApplicationSchema.parse_obj(self.get_test_config())
client.deploy_app(config)
assert client.get_serve_status().app_status.deployment_timestamp > 0
first_deploy_time = client.get_serve_status().app_status.deployment_timestamp
time.sleep(0.1)
config = self.get_test_config()
config["deployments"] = [
{
"name": "Adder",
"num_replicas": 2,
},
]
client.deploy_app(ServeApplicationSchema.parse_obj(config))
assert (
client.get_serve_status().app_status.deployment_timestamp
> first_deploy_time
)
assert client.get_serve_status().app_status.status in {
ApplicationStatus.DEPLOYING,
ApplicationStatus.RUNNING,
}
def test_serve_application_invalid_import_path(self, path):
# Test invalid import path formats
serve_application_schema = self.get_valid_serve_application_schema()
serve_application_schema["import_path"] = path
with pytest.raises(ValidationError):
ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_invalid_runtime_env(self, env):
# Test invalid runtime_env configurations
serve_application_schema = self.get_valid_serve_application_schema()
serve_application_schema["runtime_env"] = env
with pytest.raises(ValueError):
ServeApplicationSchema.parse_obj(serve_application_schema)
def test_deploy_app_runtime_env(self, client: ServeControllerClient):
config_template = {
"import_path": "conditional_dag.serve_dag",
"runtime_env": {
"working_dir": (
"https://github.com/ray-project/test_dag/archive/"
"76a741f6de31df78411b1f302071cde46f098418.zip"
)
},
}
config1 = ServeApplicationSchema.parse_obj(config_template)
client.deploy_app(config1)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "0 pizzas please!"
)
# Override the configuration
config_template["deployments"] = [
{
"name": "Adder",
"ray_actor_options": {
"runtime_env": {"env_vars": {"override_increment": "1"}}
},
}
]
config2 = ServeApplicationSchema.parse_obj(config_template)
client.deploy_app(config2)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "3 pizzas please!"
)
def test_deploy_app_update_num_replicas(self,
client: ServeControllerClient):
config = ServeApplicationSchema.parse_obj(self.get_test_config())
client.deploy_app(config)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]
).json() == "4 pizzas please!")
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 3]
).json() == "9 pizzas please!")
actors = ray.util.list_named_actors(all_namespaces=True)
config = self.get_test_config()
config["deployments"] = [
{
"name": "Adder",
"num_replicas": 2,
"user_config": {
"increment": 0,
},
"ray_actor_options": {
"num_cpus": 0.1
},
},
{
"name": "Multiplier",
"num_replicas": 3,
"user_config": {
"factor": 0,
},
"ray_actor_options": {
"num_cpus": 0.1
},
},
]
client.deploy_app(ServeApplicationSchema.parse_obj(config))
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]
).json() == "2 pizzas please!")
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 3]
).json() == "0 pizzas please!")
wait_for_condition(
lambda: client.get_serve_status().app_status.status ==
ApplicationStatus.RUNNING,
timeout=15,
)
updated_actors = ray.util.list_named_actors(all_namespaces=True)
assert len(updated_actors) == len(actors) + 3
def test_extra_fields_invalid_serve_application_schema(self):
# Undefined fields should be forbidden in the schema
serve_application_schema = self.get_valid_serve_application_schema()
# Schema should be createable with valid fields
ServeApplicationSchema.parse_obj(serve_application_schema)
# Schema should raise error when a nonspecified field is included
serve_application_schema["fake_field"] = None
with pytest.raises(ValidationError):
ServeApplicationSchema.parse_obj(serve_application_schema)
def deploy(config_file_name: str, address: str):
with open(config_file_name, "r") as config_file:
config = yaml.safe_load(config_file)
# Schematize config to validate format.
ServeApplicationSchema.parse_obj(config)
ServeSubmissionClient(address).deploy_application(config)
cli_logger.newline()
cli_logger.success(
"\nSent deploy request successfully!\n "
"* Use `serve status` to check deployments' statuses.\n "
"* Use `serve config` to see the running app's config.\n")
cli_logger.newline()
def test_deploy_app_with_overriden_config(self, client: ServeControllerClient):
config = self.get_test_config()
config["deployments"] = [
{
"name": "Multiplier",
"user_config": {
"factor": 4,
},
},
{
"name": "Adder",
"user_config": {
"increment": 5,
},
},
]
client.deploy_app(ServeApplicationSchema.parse_obj(config))
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 0]).json()
== "5 pizzas please!"
)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 2]).json()
== "8 pizzas please!"
)
def build(import_path: str, app_dir: str, output_path: Optional[str]):
sys.path.insert(0, app_dir)
node: Union[ClassNode, FunctionNode] = import_attr(import_path)
if not isinstance(node, (ClassNode, FunctionNode)):
raise TypeError(
f"Expected '{import_path}' to be ClassNode or "
f"FunctionNode, but got {type(node)}."
)
app = build_app(node)
config = ServeApplicationSchema(
deployments=[deployment_to_schema(d) for d in app.deployments.values()]
).dict()
config["import_path"] = import_path
if output_path is not None:
if not output_path.endswith(".yaml"):
raise ValueError("FILE_PATH must end with '.yaml'.")
with open(output_path, "w") as f:
yaml.safe_dump(config, stream=f, default_flow_style=False, sort_keys=False)
else:
print(yaml.safe_dump(config, default_flow_style=False, sort_keys=False), end="")
def test_serve_application_aliasing(self):
"""Check aliasing behavior for schemas."""
# Check that private options can optionally include underscore
app_dict = {
"import_path": "module.graph",
"runtime_env": {},
"deployments": [
{
"name": "d1",
"max_concurrent_queries": 3,
"autoscaling_config": {},
"_graceful_shutdown_wait_loop_s": 30,
"graceful_shutdown_timeout_s": 10,
"_health_check_period_s": 5,
"health_check_timeout_s": 7,
},
{
"name": "d2",
"max_concurrent_queries": 6,
"_autoscaling_config": {},
"graceful_shutdown_wait_loop_s": 50,
"_graceful_shutdown_timeout_s": 15,
"health_check_period_s": 53,
"_health_check_timeout_s": 73,
},
],
}
schema = ServeApplicationSchema.parse_obj(app_dict)
# Check that schema dictionary can include private options with an
# underscore (using the aliases)
private_options = {
"_autoscaling_config",
"_graceful_shutdown_wait_loop_s",
"_graceful_shutdown_timeout_s",
"_health_check_period_s",
"_health_check_timeout_s",
}
for deployment in schema.dict(by_alias=True)["deployments"]:
for option in private_options:
# Option with leading underscore
assert option in deployment
# Option without leading underscore
assert option[1:] not in deployment
# Check that schema dictionary can include private options without an
# underscore (using the field names)
for deployment in schema.dict()["deployments"]:
for option in private_options:
# Option without leading underscore
assert option[1:] in deployment
# Option with leading underscore
assert option not in deployment
def from_yaml(cls, str_or_file: Union[str, TextIO]) -> "Application":
"""Converts YAML data to deployments for an Application.
Takes in a string or a file pointer to a file containing deployment
definitions in YAML. These definitions are converted to a new
Application object containing the deployments.
To read from a file, use the following pattern:
with open("file_name.txt", "w") as f:
app = app.from_yaml(str_or_file)
Args:
str_or_file (Union[String, TextIO]): Either a string containing
YAML deployment definitions or a pointer to a file containing
YAML deployment definitions. The YAML format must adhere to the
ServeApplicationSchema JSON Schema defined in
ray.serve.schema. This function works with
Serve YAML config files.
Returns:
Application: a new Application object containing the deployments.
"""
deployments_json = yaml.safe_load(str_or_file)
schema = ServeApplicationSchema.parse_obj(deployments_json)
return cls(schema_to_serve_application(schema))
async def put_all_deployments(self, req: Request) -> Response:
from ray.serve.context import get_global_client
from ray.serve.schema import ServeApplicationSchema
config = ServeApplicationSchema.parse_obj(await req.json())
get_global_client().deploy_app(config)
return Response()
def deploy_app(self, config: ServeApplicationSchema) -> None:
ray.get(
self._controller.deploy_app.remote(
config.import_path,
config.runtime_env,
config.dict(by_alias=True, exclude_unset=True).get("deployments", []),
)
)
def run(
config_or_import_path: str,
runtime_env: str,
runtime_env_json: str,
working_dir: str,
app_dir: str,
address: str,
host: str,
port: int,
blocking: bool,
):
sys.path.insert(0, app_dir)
final_runtime_env = parse_runtime_env_args(
runtime_env=runtime_env,
runtime_env_json=runtime_env_json,
working_dir=working_dir,
)
if pathlib.Path(config_or_import_path).is_file():
config_path = config_or_import_path
cli_logger.print(f'Deploying from config file: "{config_path}".')
with open(config_path, "r") as config_file:
config = ServeApplicationSchema.parse_obj(
yaml.safe_load(config_file))
is_config = True
else:
import_path = config_or_import_path
cli_logger.print(f'Deploying from import path: "{import_path}".')
node = import_attr(import_path)
is_config = False
# Setting the runtime_env here will set defaults for the deployments.
ray.init(address=address,
namespace=SERVE_NAMESPACE,
runtime_env=final_runtime_env)
client = serve.start(detached=True)
try:
if is_config:
client.deploy_app(config)
else:
serve.run(node, host=host, port=port)
cli_logger.success("Deployed successfully.")
if blocking:
while True:
# Block, letting Ray print logs to the terminal.
time.sleep(10)
except KeyboardInterrupt:
cli_logger.info("Got KeyboardInterrupt, shutting down...")
serve.shutdown()
sys.exit()
def deploy_app(
self, config: ServeApplicationSchema, update_time: bool = True
) -> None:
"""Kicks off a task that deploys a Serve application.
Cancels any previous in-progress task that is deploying a Serve
application.
Args:
config: Contains the following:
import_path: Serve deployment graph's import path
runtime_env: runtime_env to run the deployment graph in
deployment_override_options: Dictionaries that
contain argument-value options that can be passed directly
into a set_options() call. Overrides deployment options set
in the graph's code itself.
update_time: Whether to update the deployment_timestamp.
"""
if update_time:
self.deployment_timestamp = time.time()
config_dict = config.dict(exclude_unset=True)
self.kv_store.put(
CONFIG_CHECKPOINT_KEY,
pickle.dumps((self.deployment_timestamp, config_dict)),
)
if self.config_deployment_request_ref is not None:
ray.cancel(self.config_deployment_request_ref)
logger.info(
"Received new config deployment request. Cancelling "
"previous request."
)
deployment_override_options = config.dict(
by_alias=True, exclude_unset=True
).get("deployments", [])
self.config_deployment_request_ref = run_graph.options(
runtime_env=config.runtime_env
).remote(config.import_path, config.runtime_env, deployment_override_options)
def test_deploy_app_basic(self, client: ServeControllerClient):
config = ServeApplicationSchema.parse_obj(self.get_test_config())
client.deploy_app(config)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]
).json() == "4 pizzas please!")
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 3]
).json() == "9 pizzas please!")
def to_dict(self) -> Dict:
"""Returns this Application's deployments as a dictionary.
This dictionary adheres to the Serve REST API schema. It can be deployed
via the Serve REST API.
Returns:
Dict: The Application's deployments formatted in a dictionary.
"""
return ServeApplicationSchema(deployments=[
deployment_to_schema(d) for d in self._deployments.values()
]).dict()
def from_dict(cls, d: Dict) -> "Application":
"""Converts a dictionary of deployment data to an Application.
Takes in a dictionary matching the Serve REST API schema and converts
it to an Application containing those deployments.
Args:
d (Dict): A dictionary containing the deployments' data that matches
the Serve REST API schema.
Returns:
Application: a new Application object containing the deployments.
"""
schema = ServeApplicationSchema.parse_obj(d)
return cls(schema_to_serve_application(schema))
async def put_all_deployments(self, req: Request) -> Response:
from ray import serve
from ray.serve.context import get_global_client
from ray.serve.schema import ServeApplicationSchema
from ray.serve.application import Application
config = ServeApplicationSchema.parse_obj(await req.json())
if config.import_path is not None:
client = get_global_client(_override_controller_namespace="serve")
client.deploy_app(config)
else:
# TODO (shrekris-anyscale): Remove this conditional path
app = Application.from_dict(await req.json())
serve.run(app, _blocking=False)
return Response()
def test_run_graph_task_uses_zero_cpus():
"""Check that the run_graph() task uses zero CPUs."""
ray.init(num_cpus=2)
client = serve.start(detached=True)
config = {"import_path": "ray.serve.tests.test_standalone.WaiterNode"}
config = ServeApplicationSchema.parse_obj(config)
client.deploy_app(config)
with pytest.raises(RuntimeError):
wait_for_condition(lambda: ray.available_resources()["CPU"] < 1.9,
timeout=5)
wait_for_condition(lambda: requests.get("http://localhost:8000/Waiter").
text == "May I take your order?")
serve.shutdown()
ray.shutdown()
def test_controller_recover_and_deploy(self, client: ServeControllerClient):
"""Ensure that in-progress deploy can finish even after controller dies."""
config = ServeApplicationSchema.parse_obj(self.get_test_config())
client.deploy_app(config)
# Wait for app to deploy
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "4 pizzas please!"
)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 3]).json()
== "9 pizzas please!"
)
deployment_timestamp = client.get_serve_status().app_status.deployment_timestamp
# Delete all deployments, but don't update config
client.delete_deployments(
["Router", "Multiplier", "Adder", "create_order", "DAGDriver"]
)
ray.kill(client._controller, no_restart=False)
# When controller restarts, it should redeploy config automatically
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json()
== "4 pizzas please!"
)
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MUL", 3]).json()
== "9 pizzas please!"
)
assert (
deployment_timestamp
== client.get_serve_status().app_status.deployment_timestamp
)
serve.shutdown()
client = serve.start(detached=True)
# Ensure config checkpoint has been deleted
assert client.get_serve_status().app_status.deployment_timestamp == 0
def build(import_path: str, app_dir: str, output_path: Optional[str]):
sys.path.insert(0, app_dir)
node: Union[ClassNode, FunctionNode] = import_attr(import_path)
if not isinstance(node, (ClassNode, FunctionNode)):
raise TypeError(f"Expected '{import_path}' to be ClassNode or "
f"FunctionNode, but got {type(node)}.")
app = build_app(node)
config = ServeApplicationSchema(deployments=[
deployment_to_schema(d) for d in app.deployments.values()
]).dict()
config["import_path"] = import_path
config_str = ("# This file was generated using the `serve build` command "
f"on Ray v{ray.__version__}.\n\n")
config_str += yaml.dump(config,
Dumper=ServeBuildDumper,
default_flow_style=False,
sort_keys=False)
with open(output_path, "w") if output_path else sys.stdout as f:
f.write(config_str)
def _recover_config_from_checkpoint(self):
checkpoint = self.kv_store.get(CONFIG_CHECKPOINT_KEY)
if checkpoint is not None:
self.deployment_timestamp, config = pickle.loads(checkpoint)
self.deploy_app(ServeApplicationSchema.parse_obj(config),
update_time=False)
},
{
"name": "PearStand",
"ray_actor_options": {
"num_cpus": 0.1
}
},
{
"name": "DAGDriver",
"ray_actor_options": {
"num_cpus": 0.1
}
},
],
}
client.deploy_app(ServeApplicationSchema.parse_obj(config1))
wait_for_condition(
lambda: requests.post("http://localhost:8000/", json=["MANGO", 1]).json()
== 3,
timeout=15,
)
check_fruit_deployment_graph()
config2 = {
"import_path":
"fruit.deployment_graph",
"runtime_env": {
"working_dir":
("https://github.com/ray-project/serve_config_examples/archive/HEAD.zip"
)
},
"deployments": [
def test_valid_serve_application_schema(self):
# Ensure a valid ServeApplicationSchema can be generated
serve_application_schema = self.get_valid_serve_application_schema()
ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_valid_import_path(self, path):
# Test valid import path formats
serve_application_schema = self.get_valid_serve_application_schema()
serve_application_schema["import_path"] = path
ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_valid_runtime_env(self, env):
# Test valid runtime_env configurations
serve_application_schema = self.get_valid_serve_application_schema()
serve_application_schema["runtime_env"] = env
ServeApplicationSchema.parse_obj(serve_application_schema)