async def __init__(self, backend_tag, replica_tag, init_args, backend_config: BackendConfig, controller_name: str): if isinstance(backend_def, str): backend = import_attr(backend_def) else: backend = backend_def if inspect.isfunction(backend): is_function = True elif inspect.isclass(backend): is_function = False else: assert False, ("backend_def must be function, class, or " "corresponding import path.") # Set the controller name so that serve.connect() in the user's # backend code will connect to the instance that this backend is # running in. ray.serve.api._set_internal_replica_context(backend_tag, replica_tag, controller_name, servable_object=None) if is_function: _callable = backend else: # This allows backends to define an async __init__ method # (required for FastAPI backend definition). _callable = backend.__new__(backend) await sync_to_async(_callable.__init__)(*init_args) # Setting the context again to update the servable_object. ray.serve.api._set_internal_replica_context( backend_tag, replica_tag, controller_name, servable_object=_callable) assert controller_name, "Must provide a valid controller_name" controller_handle = ray.get_actor(controller_name) self.backend = RayServeReplica(_callable, backend_config, is_function, controller_handle)
def load_plugins(self, plugin_classes: List[str]): """Load runtime env plugins""" for plugin_class_path in plugin_classes: plugin_class = import_attr(plugin_class_path) if not issubclass(plugin_class, RuntimeEnvPlugin): default_logger.warning( "Invalid runtime env plugin class %s. " "The plugin class must inherit " "ray._private.runtime_env.plugin.RuntimeEnvPlugin.", plugin_class, ) continue if not plugin_class.name: default_logger.warning( "No valid name in runtime env plugin %s", plugin_class) continue if plugin_class.name in self.plugins: default_logger.warning( "The name of runtime env plugin %s conflicts with %s", plugin_class, self.plugins[plugin_class.name], ) continue self.plugins[plugin_class.name] = plugin_class()
def load_http_adapter( http_adapter: Optional[Union[str, HTTPAdapterFn, Type[BaseModel]]] ) -> HTTPAdapterFn: if http_adapter is None: http_adapter = DEFAULT_HTTP_ADAPTER if isinstance(http_adapter, str): http_adapter = import_attr(http_adapter) if inspect.isclass(http_adapter) and issubclass(http_adapter, BaseModel): def http_adapter(inp: http_adapter = Body(...)): return inp if not inspect.isfunction(http_adapter): raise ValueError( "input schema must be a callable function or pydantic model class." ) if any(param.annotation == inspect.Parameter.empty for param in inspect.signature(http_adapter).parameters.values()): raise ValueError( "input schema function's signature should be type annotated.") return http_adapter
def build(import_path: str, app_dir: str, output_path: Optional[str]): sys.path.insert(0, app_dir) node: Union[ClassNode, FunctionNode] = import_attr(import_path) if not isinstance(node, (ClassNode, FunctionNode)): raise TypeError(f"Expected '{import_path}' to be ClassNode or " f"FunctionNode, but got {type(node)}.") app = build_app(node) config = ServeApplicationSchema(deployments=[ deployment_to_schema(d) for d in app.deployments.values() ]).dict() config["import_path"] = import_path config_str = ("# This file was generated using the `serve build` command " f"on Ray v{ray.__version__}.\n\n") config_str += yaml.dump(config, Dumper=ServeBuildDumper, default_flow_style=False, sort_keys=False) with open(output_path, "w") if output_path else sys.stdout as f: f.write(config_str)
# todo flag "--rm" container_command = [ container_driver, "run", "-v", tmp_dir + ":" + tmp_dir, "--cgroup-manager=cgroupfs", "--network=host", "--pid=host", "--ipc=host", "--env-host" ] container_command.append("--env") container_command.append("RAY_RAYLET_PID=" + str(os.getppid())) if container_option.get("run_options"): container_command.extend(container_option.get("run_options")) container_command.append("--entrypoint") container_command.append("python") container_command.append(container_option.get("image")) container_command.extend(entrypoint_args) logger.warning("start worker in container: {}".format(container_command)) os.execvp(container_driver, container_command) if __name__ == "__main__": args, remaining_args = parser.parse_known_args() runtime_env: dict = json.loads(args.serialized_runtime_env or "{}") container_option = runtime_env.get("container_option") if container_option and container_option.get("image"): start_worker_in_container(container_option, args, remaining_args) else: remaining_args.append("--serialized-runtime-env") remaining_args.append(args.serialized_runtime_env or "{}") setup = import_attr(args.worker_setup_hook) setup(remaining_args)
def run_setup_with_logger(): runtime_env = RuntimeEnv(serialized_runtime_env=serialized_runtime_env) allocated_resource: dict = json.loads( serialized_allocated_resource_instances or "{}" ) # Use a separate logger for each job. per_job_logger = self.get_or_create_logger(request.job_id) # TODO(chenk008): Add log about allocated_resource to # avoid lint error. That will be moved to cgroup plugin. per_job_logger.debug(f"Worker has resource :" f"{allocated_resource}") context = RuntimeEnvContext(env_vars=runtime_env.env_vars()) self._container_manager.setup( runtime_env, context, logger=per_job_logger ) for (manager, uri_cache) in [ (self._working_dir_manager, self._working_dir_uri_cache), (self._conda_manager, self._conda_uri_cache), (self._pip_manager, self._pip_uri_cache), ]: uri = manager.get_uri(runtime_env) if uri is not None: if uri not in uri_cache: per_job_logger.debug(f"Cache miss for URI {uri}.") size_bytes = manager.create( uri, runtime_env, context, logger=per_job_logger ) uri_cache.add(uri, size_bytes, logger=per_job_logger) else: per_job_logger.debug(f"Cache hit for URI {uri}.") uri_cache.mark_used(uri, logger=per_job_logger) manager.modify_context(uri, runtime_env, context) # Set up py_modules. For now, py_modules uses multiple URIs so # the logic is slightly different from working_dir, conda, and # pip above. py_modules_uris = self._py_modules_manager.get_uris(runtime_env) if py_modules_uris is not None: for uri in py_modules_uris: if uri not in self._py_modules_uri_cache: per_job_logger.debug(f"Cache miss for URI {uri}.") size_bytes = self._py_modules_manager.create( uri, runtime_env, context, logger=per_job_logger ) self._py_modules_uri_cache.add( uri, size_bytes, logger=per_job_logger ) else: per_job_logger.debug(f"Cache hit for URI {uri}.") self._py_modules_uri_cache.mark_used( uri, logger=per_job_logger ) self._py_modules_manager.modify_context( py_modules_uris, runtime_env, context ) # Add the mapping of URIs -> the serialized environment to be # used for cache invalidation. if runtime_env.working_dir_uri(): uri = runtime_env.working_dir_uri() self._uris_to_envs[uri].add(serialized_runtime_env) if runtime_env.py_modules_uris(): for uri in runtime_env.py_modules_uris(): self._uris_to_envs[uri].add(serialized_runtime_env) if runtime_env.conda_uri(): uri = runtime_env.conda_uri() self._uris_to_envs[uri].add(serialized_runtime_env) if runtime_env.pip_uri(): uri = runtime_env.pip_uri() self._uris_to_envs[uri].add(serialized_runtime_env) if runtime_env.plugin_uris(): for uri in runtime_env.plugin_uris(): self._uris_to_envs[uri].add(serialized_runtime_env) # Run setup function from all the plugins for plugin_class_path, config in runtime_env.plugins(): per_job_logger.debug( f"Setting up runtime env plugin {plugin_class_path}" ) plugin_class = import_attr(plugin_class_path) # TODO(simon): implement uri support plugin_class.create( "uri not implemented", json.loads(config), context ) plugin_class.modify_context( "uri not implemented", json.loads(config), context ) return context
def __init__(self, runtime_env: Dict[str, Any], _validate: bool = True): super().__init__() # Blindly trust that the runtime_env has already been validated. # This is dangerous and should only be used internally (e.g., on the # deserialization codepath. if not _validate: self.update(runtime_env) return if runtime_env.get("conda") and runtime_env.get("pip"): raise ValueError( "The 'pip' field and 'conda' field of " "runtime_env cannot both be specified.\n" f"specified pip field: {runtime_env['pip']}\n" f"specified conda field: {runtime_env['conda']}\n" "To use pip with conda, please only set the 'conda' " "field, and specify your pip dependencies " "within the conda YAML config dict: see " "https://conda.io/projects/conda/en/latest/" "user-guide/tasks/manage-environments.html" "#create-env-file-manually") for option, validate_fn in OPTION_TO_VALIDATION_FN.items(): option_val = runtime_env.get(option) if option_val is not None: validated_option_val = validate_fn(option_val) if validated_option_val is not None: self[option] = validated_option_val if "_ray_release" in runtime_env: self["_ray_release"] = runtime_env["_ray_release"] if "_ray_commit" in runtime_env: self["_ray_commit"] = runtime_env["_ray_commit"] else: if self.get("pip") or self.get("conda"): self["_ray_commit"] = ray.__commit__ # Used for testing wheels that have not yet been merged into master. # If this is set to True, then we do not inject Ray into the conda # or pip dependencies. if "_inject_current_ray" in runtime_env: self["_inject_current_ray"] = runtime_env["_inject_current_ray"] elif "RAY_RUNTIME_ENV_LOCAL_DEV_MODE" in os.environ: self["_inject_current_ray"] = True if "plugins" in runtime_env: self["plugins"] = dict() for class_path, plugin_field in runtime_env["plugins"].items(): plugin_class: RuntimeEnvPlugin = import_attr(class_path) if not issubclass(plugin_class, RuntimeEnvPlugin): # TODO(simon): move the inferface to public once ready. raise TypeError( f"{class_path} must be inherit from " "ray._private.runtime_env.plugin.RuntimeEnvPlugin.") # TODO(simon): implement uri support. _ = plugin_class.validate(runtime_env) # Validation passed, add the entry to parsed runtime env. self["plugins"][class_path] = plugin_field unknown_fields = (set(runtime_env.keys()) - ParsedRuntimeEnv.known_fields) if len(unknown_fields): logger.warning( "The following unknown entries in the runtime_env dictionary " f"will be ignored: {unknown_fields}. If you intended to use " "them as plugins, they must be nested in the `plugins` field.") # NOTE(architkulkarni): This allows worker caching code in C++ to check # if a runtime env is empty without deserializing it. This is a catch- # all; for validated inputs we won't set the key if the value is None. if all(val is None for val in self.values()): self.clear()
def create_deployment(deployment: str, options_json: str): deployment_cls = import_attr(deployment) if not isinstance(deployment_cls, Deployment): deployment_cls = serve.deployment(deployment_cls) options = json.loads(options_json) deployment_cls.options(**options).deploy()
def __init__(self, runtime_env_json: dict, working_dir: Optional[str] = None): # Simple dictionary with all options validated. This will always # contain all supported keys; values will be set to None if # unspecified. However, if all values are None this is set to {}. self._dict = dict() if "working_dir" in runtime_env_json: self._dict["working_dir"] = runtime_env_json["working_dir"] if not isinstance(self._dict["working_dir"], str): raise TypeError("`working_dir` must be a string. Type " f"{type(self._dict['working_dir'])} received.") working_dir = Path(self._dict["working_dir"]).absolute() else: self._dict["working_dir"] = None working_dir = Path(working_dir).absolute() if working_dir else None self._dict["conda"] = None if "conda" in runtime_env_json: if sys.platform == "win32": raise NotImplementedError("The 'conda' field in runtime_env " "is not currently supported on " "Windows.") conda = runtime_env_json["conda"] if isinstance(conda, str): self._dict["conda"] = conda elif isinstance(conda, dict): self._dict["conda"] = conda elif conda is not None: raise TypeError("runtime_env['conda'] must be of type str or " "dict") self._dict["pip"] = None pip = runtime_env_json.get("pip") if pip is not None: if sys.platform == "win32": raise NotImplementedError("The 'pip' field in runtime_env " "is not currently supported on " "Windows.") conda = runtime_env_json.get("conda") if runtime_env_json.get("conda") is not None: raise ValueError( "The 'pip' field and 'conda' field of " "runtime_env cannot both be specified.\n" f"specified pip field: {runtime_env_json['pip']}\n" f"specified conda field: {runtime_env_json['conda']}\n" "To use pip with conda, please only set the 'conda' " "field, and specify your pip dependencies " "within the conda YAML config dict: see " "https://conda.io/projects/conda/en/latest/" "user-guide/tasks/manage-environments.html" "#create-env-file-manually") if isinstance(pip, str): self._dict["pip"] = pip elif isinstance(pip, list) and all( isinstance(dep, str) for dep in pip): # Construct valid pip requirements.txt from list of packages. self._dict["pip"] = "\n".join(pip) + "\n" else: raise TypeError("runtime_env['pip'] must be of type str or " "List[str]") if "uris" in runtime_env_json: self._dict["uris"] = runtime_env_json["uris"] if "container" in runtime_env_json: self._dict["container"] = runtime_env_json["container"] self._dict["env_vars"] = None env_vars = runtime_env_json.get("env_vars") if env_vars is not None: self._dict["env_vars"] = env_vars if not (isinstance(env_vars, dict) and all( isinstance(k, str) and isinstance(v, str) for (k, v) in env_vars.items())): raise TypeError("runtime_env['env_vars'] must be of type" "Dict[str, str]") if "_ray_release" in runtime_env_json: self._dict["_ray_release"] = runtime_env_json["_ray_release"] if "_ray_commit" in runtime_env_json: self._dict["_ray_commit"] = runtime_env_json["_ray_commit"] else: if self._dict.get("pip") or self._dict.get("conda"): self._dict["_ray_commit"] = ray.__commit__ # Used for testing wheels that have not yet been merged into master. # If this is set to True, then we do not inject Ray into the conda # or pip dependencies. if os.environ.get("RAY_RUNTIME_ENV_LOCAL_DEV_MODE"): runtime_env_json["_inject_current_ray"] = True if "_inject_current_ray" in runtime_env_json: self._dict["_inject_current_ray"] = runtime_env_json[ "_inject_current_ray"] # TODO(ekl) we should have better schema validation here. # TODO(ekl) support py_modules # TODO(architkulkarni) support docker if "plugins" in runtime_env_json: self._dict["plugins"] = dict() for class_path, plugin_field in runtime_env_json["plugins"].items( ): plugin_class: RuntimeEnvPlugin = import_attr(class_path) if not issubclass(plugin_class, RuntimeEnvPlugin): # TODO(simon): move the inferface to public once ready. raise TypeError( f"{class_path} must be inherit from " "ray._private.runtime_env.plugin.RuntimeEnvPlugin.") # TODO(simon): implement uri support. _ = plugin_class.validate(runtime_env_json) # Validation passed, add the entry to parsed runtime env. self._dict["plugins"][class_path] = plugin_field unknown_fields = (set(runtime_env_json.keys()) - RuntimeEnvDict.known_fields) if len(unknown_fields): logger.warning( "The following unknown entries in the runtime_env dictionary " f"will be ignored: {unknown_fields}. If you are intended to " "use plugin, make sure to nest them in the ``plugins`` field.") # TODO(architkulkarni) This is to make it easy for the worker caching # code in C++ to check if the env is empty without deserializing and # parsing it. We should use a less confusing approach here. if all(val is None for val in self._dict.values()): self._dict = {}
def make_kv_store(checkpoint_path, namespace): """Create KVStore instance based on checkpoint_path configuration""" if checkpoint_path == DEFAULT_CHECKPOINT_PATH: logger.info( "Using RayInternalKVStore for controller checkpoint and recovery.") return RayInternalKVStore(namespace) else: parsed_url = urlparse(checkpoint_path) if parsed_url.scheme not in {"gs", "s3", "file", "custom"}: raise ValueError( f"Checkpoint must be one of `{DEFAULT_CHECKPOINT_PATH}`, " "`file://path...`, `gs://path...`, `s3://path...`, or " "`custom://my_module.ClassName?arg1=val1`. But it is " f"{checkpoint_path}") if parsed_url.scheme == "file": db_path = parsed_url.netloc + parsed_url.path logger.info("Using RayLocalKVStore for controller " f"checkpoint and recovery: path={db_path}") return RayLocalKVStore(namespace, db_path) if parsed_url.scheme == "gs": bucket = parsed_url.netloc # We need to strip leading "/" in path as right key to use in # gcs. Ex: gs://bucket/folder/file.zip -> key = "folder/file.zip" prefix = parsed_url.path.lstrip("/") logger.info("Using Ray GCS KVStore for controller checkpoint and" " recovery: " f"bucket={bucket} checkpoint_path={checkpoint_path}") return RayGcsKVStore( namespace, bucket=bucket, prefix=prefix, ) if parsed_url.scheme == "s3": bucket = parsed_url.netloc # We need to strip leading "/" in path as right key to use in # boto3. Ex: s3://bucket/folder/file.zip -> key = "folder/file.zip" prefix = parsed_url.path.lstrip("/") logger.info( "Using Ray S3 KVStore for controller checkpoint and recovery: " f"bucket={bucket} checkpoint_path={checkpoint_path}") return RayS3KVStore( namespace, bucket=bucket, prefix=prefix, ) if parsed_url.scheme == "custom": kwargs = dict(parse_qsl(parsed_url.query)) # Prepare the parameters to initialize imported class. checkpoint_provider = parsed_url.netloc KVStoreClass = import_attr(checkpoint_provider) if not issubclass(KVStoreClass, KVStoreBase): raise ValueError( f"{KVStoreClass} doesn't inherit from " "`ray.serve.storage.kv_store_base.KVStoreBase`.") logger.info( f"Using {checkpoint_provider} for controller checkpoint and " f"recovery: kwargs={kwargs}") return KVStoreClass(namespace=namespace, **kwargs) raise RuntimeError("This shouldn't be reachable.")
def __init__(self, dashboard_agent): super().__init__(dashboard_agent) self._session_dir = dashboard_agent.session_dir self._runtime_env_dir = dashboard_agent.runtime_env_dir self._setup = import_attr(dashboard_agent.runtime_env_setup_hook) runtime_env.PKG_DIR = dashboard_agent.runtime_env_dir