def test_deploy_app_update_config(self, client: ServeControllerClient): config = ServeApplicationSchema.parse_obj(self.get_test_config()) client.deploy_app(config) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "4 pizzas please!" ) config = self.get_test_config() config["deployments"] = [ { "name": "Adder", "user_config": { "increment": -1, }, }, ] client.deploy_app(ServeApplicationSchema.parse_obj(config)) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "1 pizzas please!" )
def test_deploy_app_overwrite_apps(self, client: ServeControllerClient): """Check that overwriting a live app with a new one works.""" # Launch first graph. Its driver's route_prefix should be "/". test_config_1 = ServeApplicationSchema.parse_obj( { "import_path": "ray.serve.tests.test_config_files.world.DagNode", } ) client.deploy_app(test_config_1) wait_for_condition( lambda: requests.get("http://localhost:8000/").text == "wonderful world" ) # Launch second graph. Its driver's route_prefix should also be "/". # "/" should lead to the new driver. test_config_2 = ServeApplicationSchema.parse_obj( { "import_path": "ray.serve.tests.test_config_files.pizza.serve_dag", } ) client.deploy_app(test_config_2) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "4 pizzas please!" )
def test_deploy_app_update_timestamp(self, client: ServeControllerClient): assert client.get_serve_status().app_status.deployment_timestamp == 0 config = ServeApplicationSchema.parse_obj(self.get_test_config()) client.deploy_app(config) assert client.get_serve_status().app_status.deployment_timestamp > 0 first_deploy_time = client.get_serve_status().app_status.deployment_timestamp time.sleep(0.1) config = self.get_test_config() config["deployments"] = [ { "name": "Adder", "num_replicas": 2, }, ] client.deploy_app(ServeApplicationSchema.parse_obj(config)) assert ( client.get_serve_status().app_status.deployment_timestamp > first_deploy_time ) assert client.get_serve_status().app_status.status in { ApplicationStatus.DEPLOYING, ApplicationStatus.RUNNING, }
def test_serve_application_invalid_import_path(self, path): # Test invalid import path formats serve_application_schema = self.get_valid_serve_application_schema() serve_application_schema["import_path"] = path with pytest.raises(ValidationError): ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_invalid_runtime_env(self, env): # Test invalid runtime_env configurations serve_application_schema = self.get_valid_serve_application_schema() serve_application_schema["runtime_env"] = env with pytest.raises(ValueError): ServeApplicationSchema.parse_obj(serve_application_schema)
def test_deploy_app_runtime_env(self, client: ServeControllerClient): config_template = { "import_path": "conditional_dag.serve_dag", "runtime_env": { "working_dir": ( "https://github.com/ray-project/test_dag/archive/" "76a741f6de31df78411b1f302071cde46f098418.zip" ) }, } config1 = ServeApplicationSchema.parse_obj(config_template) client.deploy_app(config1) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "0 pizzas please!" ) # Override the configuration config_template["deployments"] = [ { "name": "Adder", "ray_actor_options": { "runtime_env": {"env_vars": {"override_increment": "1"}} }, } ] config2 = ServeApplicationSchema.parse_obj(config_template) client.deploy_app(config2) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "3 pizzas please!" )
def test_deploy_app_update_num_replicas(self, client: ServeControllerClient): config = ServeApplicationSchema.parse_obj(self.get_test_config()) client.deploy_app(config) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2] ).json() == "4 pizzas please!") wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 3] ).json() == "9 pizzas please!") actors = ray.util.list_named_actors(all_namespaces=True) config = self.get_test_config() config["deployments"] = [ { "name": "Adder", "num_replicas": 2, "user_config": { "increment": 0, }, "ray_actor_options": { "num_cpus": 0.1 }, }, { "name": "Multiplier", "num_replicas": 3, "user_config": { "factor": 0, }, "ray_actor_options": { "num_cpus": 0.1 }, }, ] client.deploy_app(ServeApplicationSchema.parse_obj(config)) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2] ).json() == "2 pizzas please!") wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 3] ).json() == "0 pizzas please!") wait_for_condition( lambda: client.get_serve_status().app_status.status == ApplicationStatus.RUNNING, timeout=15, ) updated_actors = ray.util.list_named_actors(all_namespaces=True) assert len(updated_actors) == len(actors) + 3
def test_extra_fields_invalid_serve_application_schema(self): # Undefined fields should be forbidden in the schema serve_application_schema = self.get_valid_serve_application_schema() # Schema should be createable with valid fields ServeApplicationSchema.parse_obj(serve_application_schema) # Schema should raise error when a nonspecified field is included serve_application_schema["fake_field"] = None with pytest.raises(ValidationError): ServeApplicationSchema.parse_obj(serve_application_schema)
def deploy(config_file_name: str, address: str): with open(config_file_name, "r") as config_file: config = yaml.safe_load(config_file) # Schematize config to validate format. ServeApplicationSchema.parse_obj(config) ServeSubmissionClient(address).deploy_application(config) cli_logger.newline() cli_logger.success( "\nSent deploy request successfully!\n " "* Use `serve status` to check deployments' statuses.\n " "* Use `serve config` to see the running app's config.\n") cli_logger.newline()
def test_deploy_app_with_overriden_config(self, client: ServeControllerClient): config = self.get_test_config() config["deployments"] = [ { "name": "Multiplier", "user_config": { "factor": 4, }, }, { "name": "Adder", "user_config": { "increment": 5, }, }, ] client.deploy_app(ServeApplicationSchema.parse_obj(config)) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 0]).json() == "5 pizzas please!" ) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 2]).json() == "8 pizzas please!" )
def build(import_path: str, app_dir: str, output_path: Optional[str]): sys.path.insert(0, app_dir) node: Union[ClassNode, FunctionNode] = import_attr(import_path) if not isinstance(node, (ClassNode, FunctionNode)): raise TypeError( f"Expected '{import_path}' to be ClassNode or " f"FunctionNode, but got {type(node)}." ) app = build_app(node) config = ServeApplicationSchema( deployments=[deployment_to_schema(d) for d in app.deployments.values()] ).dict() config["import_path"] = import_path if output_path is not None: if not output_path.endswith(".yaml"): raise ValueError("FILE_PATH must end with '.yaml'.") with open(output_path, "w") as f: yaml.safe_dump(config, stream=f, default_flow_style=False, sort_keys=False) else: print(yaml.safe_dump(config, default_flow_style=False, sort_keys=False), end="")
def test_serve_application_aliasing(self): """Check aliasing behavior for schemas.""" # Check that private options can optionally include underscore app_dict = { "import_path": "module.graph", "runtime_env": {}, "deployments": [ { "name": "d1", "max_concurrent_queries": 3, "autoscaling_config": {}, "_graceful_shutdown_wait_loop_s": 30, "graceful_shutdown_timeout_s": 10, "_health_check_period_s": 5, "health_check_timeout_s": 7, }, { "name": "d2", "max_concurrent_queries": 6, "_autoscaling_config": {}, "graceful_shutdown_wait_loop_s": 50, "_graceful_shutdown_timeout_s": 15, "health_check_period_s": 53, "_health_check_timeout_s": 73, }, ], } schema = ServeApplicationSchema.parse_obj(app_dict) # Check that schema dictionary can include private options with an # underscore (using the aliases) private_options = { "_autoscaling_config", "_graceful_shutdown_wait_loop_s", "_graceful_shutdown_timeout_s", "_health_check_period_s", "_health_check_timeout_s", } for deployment in schema.dict(by_alias=True)["deployments"]: for option in private_options: # Option with leading underscore assert option in deployment # Option without leading underscore assert option[1:] not in deployment # Check that schema dictionary can include private options without an # underscore (using the field names) for deployment in schema.dict()["deployments"]: for option in private_options: # Option without leading underscore assert option[1:] in deployment # Option with leading underscore assert option not in deployment
def from_yaml(cls, str_or_file: Union[str, TextIO]) -> "Application": """Converts YAML data to deployments for an Application. Takes in a string or a file pointer to a file containing deployment definitions in YAML. These definitions are converted to a new Application object containing the deployments. To read from a file, use the following pattern: with open("file_name.txt", "w") as f: app = app.from_yaml(str_or_file) Args: str_or_file (Union[String, TextIO]): Either a string containing YAML deployment definitions or a pointer to a file containing YAML deployment definitions. The YAML format must adhere to the ServeApplicationSchema JSON Schema defined in ray.serve.schema. This function works with Serve YAML config files. Returns: Application: a new Application object containing the deployments. """ deployments_json = yaml.safe_load(str_or_file) schema = ServeApplicationSchema.parse_obj(deployments_json) return cls(schema_to_serve_application(schema))
async def put_all_deployments(self, req: Request) -> Response: from ray.serve.context import get_global_client from ray.serve.schema import ServeApplicationSchema config = ServeApplicationSchema.parse_obj(await req.json()) get_global_client().deploy_app(config) return Response()
def deploy_app(self, config: ServeApplicationSchema) -> None: ray.get( self._controller.deploy_app.remote( config.import_path, config.runtime_env, config.dict(by_alias=True, exclude_unset=True).get("deployments", []), ) )
def run( config_or_import_path: str, runtime_env: str, runtime_env_json: str, working_dir: str, app_dir: str, address: str, host: str, port: int, blocking: bool, ): sys.path.insert(0, app_dir) final_runtime_env = parse_runtime_env_args( runtime_env=runtime_env, runtime_env_json=runtime_env_json, working_dir=working_dir, ) if pathlib.Path(config_or_import_path).is_file(): config_path = config_or_import_path cli_logger.print(f'Deploying from config file: "{config_path}".') with open(config_path, "r") as config_file: config = ServeApplicationSchema.parse_obj( yaml.safe_load(config_file)) is_config = True else: import_path = config_or_import_path cli_logger.print(f'Deploying from import path: "{import_path}".') node = import_attr(import_path) is_config = False # Setting the runtime_env here will set defaults for the deployments. ray.init(address=address, namespace=SERVE_NAMESPACE, runtime_env=final_runtime_env) client = serve.start(detached=True) try: if is_config: client.deploy_app(config) else: serve.run(node, host=host, port=port) cli_logger.success("Deployed successfully.") if blocking: while True: # Block, letting Ray print logs to the terminal. time.sleep(10) except KeyboardInterrupt: cli_logger.info("Got KeyboardInterrupt, shutting down...") serve.shutdown() sys.exit()
def deploy_app( self, config: ServeApplicationSchema, update_time: bool = True ) -> None: """Kicks off a task that deploys a Serve application. Cancels any previous in-progress task that is deploying a Serve application. Args: config: Contains the following: import_path: Serve deployment graph's import path runtime_env: runtime_env to run the deployment graph in deployment_override_options: Dictionaries that contain argument-value options that can be passed directly into a set_options() call. Overrides deployment options set in the graph's code itself. update_time: Whether to update the deployment_timestamp. """ if update_time: self.deployment_timestamp = time.time() config_dict = config.dict(exclude_unset=True) self.kv_store.put( CONFIG_CHECKPOINT_KEY, pickle.dumps((self.deployment_timestamp, config_dict)), ) if self.config_deployment_request_ref is not None: ray.cancel(self.config_deployment_request_ref) logger.info( "Received new config deployment request. Cancelling " "previous request." ) deployment_override_options = config.dict( by_alias=True, exclude_unset=True ).get("deployments", []) self.config_deployment_request_ref = run_graph.options( runtime_env=config.runtime_env ).remote(config.import_path, config.runtime_env, deployment_override_options)
def test_deploy_app_basic(self, client: ServeControllerClient): config = ServeApplicationSchema.parse_obj(self.get_test_config()) client.deploy_app(config) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2] ).json() == "4 pizzas please!") wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 3] ).json() == "9 pizzas please!")
def to_dict(self) -> Dict: """Returns this Application's deployments as a dictionary. This dictionary adheres to the Serve REST API schema. It can be deployed via the Serve REST API. Returns: Dict: The Application's deployments formatted in a dictionary. """ return ServeApplicationSchema(deployments=[ deployment_to_schema(d) for d in self._deployments.values() ]).dict()
def from_dict(cls, d: Dict) -> "Application": """Converts a dictionary of deployment data to an Application. Takes in a dictionary matching the Serve REST API schema and converts it to an Application containing those deployments. Args: d (Dict): A dictionary containing the deployments' data that matches the Serve REST API schema. Returns: Application: a new Application object containing the deployments. """ schema = ServeApplicationSchema.parse_obj(d) return cls(schema_to_serve_application(schema))
async def put_all_deployments(self, req: Request) -> Response: from ray import serve from ray.serve.context import get_global_client from ray.serve.schema import ServeApplicationSchema from ray.serve.application import Application config = ServeApplicationSchema.parse_obj(await req.json()) if config.import_path is not None: client = get_global_client(_override_controller_namespace="serve") client.deploy_app(config) else: # TODO (shrekris-anyscale): Remove this conditional path app = Application.from_dict(await req.json()) serve.run(app, _blocking=False) return Response()
def test_run_graph_task_uses_zero_cpus(): """Check that the run_graph() task uses zero CPUs.""" ray.init(num_cpus=2) client = serve.start(detached=True) config = {"import_path": "ray.serve.tests.test_standalone.WaiterNode"} config = ServeApplicationSchema.parse_obj(config) client.deploy_app(config) with pytest.raises(RuntimeError): wait_for_condition(lambda: ray.available_resources()["CPU"] < 1.9, timeout=5) wait_for_condition(lambda: requests.get("http://localhost:8000/Waiter"). text == "May I take your order?") serve.shutdown() ray.shutdown()
def test_controller_recover_and_deploy(self, client: ServeControllerClient): """Ensure that in-progress deploy can finish even after controller dies.""" config = ServeApplicationSchema.parse_obj(self.get_test_config()) client.deploy_app(config) # Wait for app to deploy wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "4 pizzas please!" ) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 3]).json() == "9 pizzas please!" ) deployment_timestamp = client.get_serve_status().app_status.deployment_timestamp # Delete all deployments, but don't update config client.delete_deployments( ["Router", "Multiplier", "Adder", "create_order", "DAGDriver"] ) ray.kill(client._controller, no_restart=False) # When controller restarts, it should redeploy config automatically wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["ADD", 2]).json() == "4 pizzas please!" ) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MUL", 3]).json() == "9 pizzas please!" ) assert ( deployment_timestamp == client.get_serve_status().app_status.deployment_timestamp ) serve.shutdown() client = serve.start(detached=True) # Ensure config checkpoint has been deleted assert client.get_serve_status().app_status.deployment_timestamp == 0
def build(import_path: str, app_dir: str, output_path: Optional[str]): sys.path.insert(0, app_dir) node: Union[ClassNode, FunctionNode] = import_attr(import_path) if not isinstance(node, (ClassNode, FunctionNode)): raise TypeError(f"Expected '{import_path}' to be ClassNode or " f"FunctionNode, but got {type(node)}.") app = build_app(node) config = ServeApplicationSchema(deployments=[ deployment_to_schema(d) for d in app.deployments.values() ]).dict() config["import_path"] = import_path config_str = ("# This file was generated using the `serve build` command " f"on Ray v{ray.__version__}.\n\n") config_str += yaml.dump(config, Dumper=ServeBuildDumper, default_flow_style=False, sort_keys=False) with open(output_path, "w") if output_path else sys.stdout as f: f.write(config_str)
def _recover_config_from_checkpoint(self): checkpoint = self.kv_store.get(CONFIG_CHECKPOINT_KEY) if checkpoint is not None: self.deployment_timestamp, config = pickle.loads(checkpoint) self.deploy_app(ServeApplicationSchema.parse_obj(config), update_time=False)
}, { "name": "PearStand", "ray_actor_options": { "num_cpus": 0.1 } }, { "name": "DAGDriver", "ray_actor_options": { "num_cpus": 0.1 } }, ], } client.deploy_app(ServeApplicationSchema.parse_obj(config1)) wait_for_condition( lambda: requests.post("http://localhost:8000/", json=["MANGO", 1]).json() == 3, timeout=15, ) check_fruit_deployment_graph() config2 = { "import_path": "fruit.deployment_graph", "runtime_env": { "working_dir": ("https://github.com/ray-project/serve_config_examples/archive/HEAD.zip" ) }, "deployments": [
def test_valid_serve_application_schema(self): # Ensure a valid ServeApplicationSchema can be generated serve_application_schema = self.get_valid_serve_application_schema() ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_valid_import_path(self, path): # Test valid import path formats serve_application_schema = self.get_valid_serve_application_schema() serve_application_schema["import_path"] = path ServeApplicationSchema.parse_obj(serve_application_schema)
def test_serve_application_valid_runtime_env(self, env): # Test valid runtime_env configurations serve_application_schema = self.get_valid_serve_application_schema() serve_application_schema["runtime_env"] = env ServeApplicationSchema.parse_obj(serve_application_schema)