async def __init__(self, backend_tag, replica_tag, init_args,
backend_config: BackendConfig,
controller_name: str):
if isinstance(backend_def, str):
backend = import_attr(backend_def)
else:
backend = backend_def
if inspect.isfunction(backend):
is_function = True
elif inspect.isclass(backend):
is_function = False
else:
assert False, ("backend_def must be function, class, or "
"corresponding import path.")
# Set the controller name so that serve.connect() in the user's
# backend code will connect to the instance that this backend is
# running in.
ray.serve.api._set_internal_replica_context(backend_tag,
replica_tag,
controller_name,
servable_object=None)
if is_function:
_callable = backend
else:
# This allows backends to define an async __init__ method
# (required for FastAPI backend definition).
_callable = backend.__new__(backend)
await sync_to_async(_callable.__init__)(*init_args)
# Setting the context again to update the servable_object.
ray.serve.api._set_internal_replica_context(
backend_tag,
replica_tag,
controller_name,
servable_object=_callable)
assert controller_name, "Must provide a valid controller_name"
controller_handle = ray.get_actor(controller_name)
self.backend = RayServeReplica(_callable, backend_config,
is_function, controller_handle)
def load_plugins(self, plugin_classes: List[str]):
"""Load runtime env plugins"""
for plugin_class_path in plugin_classes:
plugin_class = import_attr(plugin_class_path)
if not issubclass(plugin_class, RuntimeEnvPlugin):
default_logger.warning(
"Invalid runtime env plugin class %s. "
"The plugin class must inherit "
"ray._private.runtime_env.plugin.RuntimeEnvPlugin.",
plugin_class,
)
continue
if not plugin_class.name:
default_logger.warning(
"No valid name in runtime env plugin %s", plugin_class)
continue
if plugin_class.name in self.plugins:
default_logger.warning(
"The name of runtime env plugin %s conflicts with %s",
plugin_class,
self.plugins[plugin_class.name],
)
continue
self.plugins[plugin_class.name] = plugin_class()
def load_http_adapter(
http_adapter: Optional[Union[str, HTTPAdapterFn, Type[BaseModel]]]
) -> HTTPAdapterFn:
if http_adapter is None:
http_adapter = DEFAULT_HTTP_ADAPTER
if isinstance(http_adapter, str):
http_adapter = import_attr(http_adapter)
if inspect.isclass(http_adapter) and issubclass(http_adapter, BaseModel):
def http_adapter(inp: http_adapter = Body(...)):
return inp
if not inspect.isfunction(http_adapter):
raise ValueError(
"input schema must be a callable function or pydantic model class."
)
if any(param.annotation == inspect.Parameter.empty
for param in inspect.signature(http_adapter).parameters.values()):
raise ValueError(
"input schema function's signature should be type annotated.")
return http_adapter
def build(import_path: str, app_dir: str, output_path: Optional[str]):
sys.path.insert(0, app_dir)
node: Union[ClassNode, FunctionNode] = import_attr(import_path)
if not isinstance(node, (ClassNode, FunctionNode)):
raise TypeError(f"Expected '{import_path}' to be ClassNode or "
f"FunctionNode, but got {type(node)}.")
app = build_app(node)
config = ServeApplicationSchema(deployments=[
deployment_to_schema(d) for d in app.deployments.values()
]).dict()
config["import_path"] = import_path
config_str = ("# This file was generated using the `serve build` command "
f"on Ray v{ray.__version__}.\n\n")
config_str += yaml.dump(config,
Dumper=ServeBuildDumper,
default_flow_style=False,
sort_keys=False)
with open(output_path, "w") if output_path else sys.stdout as f:
f.write(config_str)
# todo flag "--rm"
container_command = [
container_driver, "run", "-v", tmp_dir + ":" + tmp_dir,
"--cgroup-manager=cgroupfs", "--network=host", "--pid=host",
"--ipc=host", "--env-host"
]
container_command.append("--env")
container_command.append("RAY_RAYLET_PID=" + str(os.getppid()))
if container_option.get("run_options"):
container_command.extend(container_option.get("run_options"))
container_command.append("--entrypoint")
container_command.append("python")
container_command.append(container_option.get("image"))
container_command.extend(entrypoint_args)
logger.warning("start worker in container: {}".format(container_command))
os.execvp(container_driver, container_command)
if __name__ == "__main__":
args, remaining_args = parser.parse_known_args()
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
container_option = runtime_env.get("container_option")
if container_option and container_option.get("image"):
start_worker_in_container(container_option, args, remaining_args)
else:
remaining_args.append("--serialized-runtime-env")
remaining_args.append(args.serialized_runtime_env or "{}")
setup = import_attr(args.worker_setup_hook)
setup(remaining_args)
def run_setup_with_logger():
runtime_env = RuntimeEnv(serialized_runtime_env=serialized_runtime_env)
allocated_resource: dict = json.loads(
serialized_allocated_resource_instances or "{}"
)
# Use a separate logger for each job.
per_job_logger = self.get_or_create_logger(request.job_id)
# TODO(chenk008): Add log about allocated_resource to
# avoid lint error. That will be moved to cgroup plugin.
per_job_logger.debug(f"Worker has resource :" f"{allocated_resource}")
context = RuntimeEnvContext(env_vars=runtime_env.env_vars())
self._container_manager.setup(
runtime_env, context, logger=per_job_logger
)
for (manager, uri_cache) in [
(self._working_dir_manager, self._working_dir_uri_cache),
(self._conda_manager, self._conda_uri_cache),
(self._pip_manager, self._pip_uri_cache),
]:
uri = manager.get_uri(runtime_env)
if uri is not None:
if uri not in uri_cache:
per_job_logger.debug(f"Cache miss for URI {uri}.")
size_bytes = manager.create(
uri, runtime_env, context, logger=per_job_logger
)
uri_cache.add(uri, size_bytes, logger=per_job_logger)
else:
per_job_logger.debug(f"Cache hit for URI {uri}.")
uri_cache.mark_used(uri, logger=per_job_logger)
manager.modify_context(uri, runtime_env, context)
# Set up py_modules. For now, py_modules uses multiple URIs so
# the logic is slightly different from working_dir, conda, and
# pip above.
py_modules_uris = self._py_modules_manager.get_uris(runtime_env)
if py_modules_uris is not None:
for uri in py_modules_uris:
if uri not in self._py_modules_uri_cache:
per_job_logger.debug(f"Cache miss for URI {uri}.")
size_bytes = self._py_modules_manager.create(
uri, runtime_env, context, logger=per_job_logger
)
self._py_modules_uri_cache.add(
uri, size_bytes, logger=per_job_logger
)
else:
per_job_logger.debug(f"Cache hit for URI {uri}.")
self._py_modules_uri_cache.mark_used(
uri, logger=per_job_logger
)
self._py_modules_manager.modify_context(
py_modules_uris, runtime_env, context
)
# Add the mapping of URIs -> the serialized environment to be
# used for cache invalidation.
if runtime_env.working_dir_uri():
uri = runtime_env.working_dir_uri()
self._uris_to_envs[uri].add(serialized_runtime_env)
if runtime_env.py_modules_uris():
for uri in runtime_env.py_modules_uris():
self._uris_to_envs[uri].add(serialized_runtime_env)
if runtime_env.conda_uri():
uri = runtime_env.conda_uri()
self._uris_to_envs[uri].add(serialized_runtime_env)
if runtime_env.pip_uri():
uri = runtime_env.pip_uri()
self._uris_to_envs[uri].add(serialized_runtime_env)
if runtime_env.plugin_uris():
for uri in runtime_env.plugin_uris():
self._uris_to_envs[uri].add(serialized_runtime_env)
# Run setup function from all the plugins
for plugin_class_path, config in runtime_env.plugins():
per_job_logger.debug(
f"Setting up runtime env plugin {plugin_class_path}"
)
plugin_class = import_attr(plugin_class_path)
# TODO(simon): implement uri support
plugin_class.create(
"uri not implemented", json.loads(config), context
)
plugin_class.modify_context(
"uri not implemented", json.loads(config), context
)
return context
def __init__(self, runtime_env: Dict[str, Any], _validate: bool = True):
super().__init__()
# Blindly trust that the runtime_env has already been validated.
# This is dangerous and should only be used internally (e.g., on the
# deserialization codepath.
if not _validate:
self.update(runtime_env)
return
if runtime_env.get("conda") and runtime_env.get("pip"):
raise ValueError(
"The 'pip' field and 'conda' field of "
"runtime_env cannot both be specified.\n"
f"specified pip field: {runtime_env['pip']}\n"
f"specified conda field: {runtime_env['conda']}\n"
"To use pip with conda, please only set the 'conda' "
"field, and specify your pip dependencies "
"within the conda YAML config dict: see "
"https://conda.io/projects/conda/en/latest/"
"user-guide/tasks/manage-environments.html"
"#create-env-file-manually")
for option, validate_fn in OPTION_TO_VALIDATION_FN.items():
option_val = runtime_env.get(option)
if option_val is not None:
validated_option_val = validate_fn(option_val)
if validated_option_val is not None:
self[option] = validated_option_val
if "_ray_release" in runtime_env:
self["_ray_release"] = runtime_env["_ray_release"]
if "_ray_commit" in runtime_env:
self["_ray_commit"] = runtime_env["_ray_commit"]
else:
if self.get("pip") or self.get("conda"):
self["_ray_commit"] = ray.__commit__
# Used for testing wheels that have not yet been merged into master.
# If this is set to True, then we do not inject Ray into the conda
# or pip dependencies.
if "_inject_current_ray" in runtime_env:
self["_inject_current_ray"] = runtime_env["_inject_current_ray"]
elif "RAY_RUNTIME_ENV_LOCAL_DEV_MODE" in os.environ:
self["_inject_current_ray"] = True
if "plugins" in runtime_env:
self["plugins"] = dict()
for class_path, plugin_field in runtime_env["plugins"].items():
plugin_class: RuntimeEnvPlugin = import_attr(class_path)
if not issubclass(plugin_class, RuntimeEnvPlugin):
# TODO(simon): move the inferface to public once ready.
raise TypeError(
f"{class_path} must be inherit from "
"ray._private.runtime_env.plugin.RuntimeEnvPlugin.")
# TODO(simon): implement uri support.
_ = plugin_class.validate(runtime_env)
# Validation passed, add the entry to parsed runtime env.
self["plugins"][class_path] = plugin_field
unknown_fields = (set(runtime_env.keys()) -
ParsedRuntimeEnv.known_fields)
if len(unknown_fields):
logger.warning(
"The following unknown entries in the runtime_env dictionary "
f"will be ignored: {unknown_fields}. If you intended to use "
"them as plugins, they must be nested in the `plugins` field.")
# NOTE(architkulkarni): This allows worker caching code in C++ to check
# if a runtime env is empty without deserializing it. This is a catch-
# all; for validated inputs we won't set the key if the value is None.
if all(val is None for val in self.values()):
self.clear()
def create_deployment(deployment: str, options_json: str):
deployment_cls = import_attr(deployment)
if not isinstance(deployment_cls, Deployment):
deployment_cls = serve.deployment(deployment_cls)
options = json.loads(options_json)
deployment_cls.options(**options).deploy()
def __init__(self,
runtime_env_json: dict,
working_dir: Optional[str] = None):
# Simple dictionary with all options validated. This will always
# contain all supported keys; values will be set to None if
# unspecified. However, if all values are None this is set to {}.
self._dict = dict()
if "working_dir" in runtime_env_json:
self._dict["working_dir"] = runtime_env_json["working_dir"]
if not isinstance(self._dict["working_dir"], str):
raise TypeError("`working_dir` must be a string. Type "
f"{type(self._dict['working_dir'])} received.")
working_dir = Path(self._dict["working_dir"]).absolute()
else:
self._dict["working_dir"] = None
working_dir = Path(working_dir).absolute() if working_dir else None
self._dict["conda"] = None
if "conda" in runtime_env_json:
if sys.platform == "win32":
raise NotImplementedError("The 'conda' field in runtime_env "
"is not currently supported on "
"Windows.")
conda = runtime_env_json["conda"]
if isinstance(conda, str):
self._dict["conda"] = conda
elif isinstance(conda, dict):
self._dict["conda"] = conda
elif conda is not None:
raise TypeError("runtime_env['conda'] must be of type str or "
"dict")
self._dict["pip"] = None
pip = runtime_env_json.get("pip")
if pip is not None:
if sys.platform == "win32":
raise NotImplementedError("The 'pip' field in runtime_env "
"is not currently supported on "
"Windows.")
conda = runtime_env_json.get("conda")
if runtime_env_json.get("conda") is not None:
raise ValueError(
"The 'pip' field and 'conda' field of "
"runtime_env cannot both be specified.\n"
f"specified pip field: {runtime_env_json['pip']}\n"
f"specified conda field: {runtime_env_json['conda']}\n"
"To use pip with conda, please only set the 'conda' "
"field, and specify your pip dependencies "
"within the conda YAML config dict: see "
"https://conda.io/projects/conda/en/latest/"
"user-guide/tasks/manage-environments.html"
"#create-env-file-manually")
if isinstance(pip, str):
self._dict["pip"] = pip
elif isinstance(pip, list) and all(
isinstance(dep, str) for dep in pip):
# Construct valid pip requirements.txt from list of packages.
self._dict["pip"] = "\n".join(pip) + "\n"
else:
raise TypeError("runtime_env['pip'] must be of type str or "
"List[str]")
if "uris" in runtime_env_json:
self._dict["uris"] = runtime_env_json["uris"]
if "container" in runtime_env_json:
self._dict["container"] = runtime_env_json["container"]
self._dict["env_vars"] = None
env_vars = runtime_env_json.get("env_vars")
if env_vars is not None:
self._dict["env_vars"] = env_vars
if not (isinstance(env_vars, dict) and all(
isinstance(k, str) and isinstance(v, str)
for (k, v) in env_vars.items())):
raise TypeError("runtime_env['env_vars'] must be of type"
"Dict[str, str]")
if "_ray_release" in runtime_env_json:
self._dict["_ray_release"] = runtime_env_json["_ray_release"]
if "_ray_commit" in runtime_env_json:
self._dict["_ray_commit"] = runtime_env_json["_ray_commit"]
else:
if self._dict.get("pip") or self._dict.get("conda"):
self._dict["_ray_commit"] = ray.__commit__
# Used for testing wheels that have not yet been merged into master.
# If this is set to True, then we do not inject Ray into the conda
# or pip dependencies.
if os.environ.get("RAY_RUNTIME_ENV_LOCAL_DEV_MODE"):
runtime_env_json["_inject_current_ray"] = True
if "_inject_current_ray" in runtime_env_json:
self._dict["_inject_current_ray"] = runtime_env_json[
"_inject_current_ray"]
# TODO(ekl) we should have better schema validation here.
# TODO(ekl) support py_modules
# TODO(architkulkarni) support docker
if "plugins" in runtime_env_json:
self._dict["plugins"] = dict()
for class_path, plugin_field in runtime_env_json["plugins"].items(
):
plugin_class: RuntimeEnvPlugin = import_attr(class_path)
if not issubclass(plugin_class, RuntimeEnvPlugin):
# TODO(simon): move the inferface to public once ready.
raise TypeError(
f"{class_path} must be inherit from "
"ray._private.runtime_env.plugin.RuntimeEnvPlugin.")
# TODO(simon): implement uri support.
_ = plugin_class.validate(runtime_env_json)
# Validation passed, add the entry to parsed runtime env.
self._dict["plugins"][class_path] = plugin_field
unknown_fields = (set(runtime_env_json.keys()) -
RuntimeEnvDict.known_fields)
if len(unknown_fields):
logger.warning(
"The following unknown entries in the runtime_env dictionary "
f"will be ignored: {unknown_fields}. If you are intended to "
"use plugin, make sure to nest them in the ``plugins`` field.")
# TODO(architkulkarni) This is to make it easy for the worker caching
# code in C++ to check if the env is empty without deserializing and
# parsing it. We should use a less confusing approach here.
if all(val is None for val in self._dict.values()):
self._dict = {}
def make_kv_store(checkpoint_path, namespace):
"""Create KVStore instance based on checkpoint_path configuration"""
if checkpoint_path == DEFAULT_CHECKPOINT_PATH:
logger.info(
"Using RayInternalKVStore for controller checkpoint and recovery.")
return RayInternalKVStore(namespace)
else:
parsed_url = urlparse(checkpoint_path)
if parsed_url.scheme not in {"gs", "s3", "file", "custom"}:
raise ValueError(
f"Checkpoint must be one of `{DEFAULT_CHECKPOINT_PATH}`, "
"`file://path...`, `gs://path...`, `s3://path...`, or "
"`custom://my_module.ClassName?arg1=val1`. But it is "
f"{checkpoint_path}")
if parsed_url.scheme == "file":
db_path = parsed_url.netloc + parsed_url.path
logger.info("Using RayLocalKVStore for controller "
f"checkpoint and recovery: path={db_path}")
return RayLocalKVStore(namespace, db_path)
if parsed_url.scheme == "gs":
bucket = parsed_url.netloc
# We need to strip leading "/" in path as right key to use in
# gcs. Ex: gs://bucket/folder/file.zip -> key = "folder/file.zip"
prefix = parsed_url.path.lstrip("/")
logger.info("Using Ray GCS KVStore for controller checkpoint and"
" recovery: "
f"bucket={bucket} checkpoint_path={checkpoint_path}")
return RayGcsKVStore(
namespace,
bucket=bucket,
prefix=prefix,
)
if parsed_url.scheme == "s3":
bucket = parsed_url.netloc
# We need to strip leading "/" in path as right key to use in
# boto3. Ex: s3://bucket/folder/file.zip -> key = "folder/file.zip"
prefix = parsed_url.path.lstrip("/")
logger.info(
"Using Ray S3 KVStore for controller checkpoint and recovery: "
f"bucket={bucket} checkpoint_path={checkpoint_path}")
return RayS3KVStore(
namespace,
bucket=bucket,
prefix=prefix,
)
if parsed_url.scheme == "custom":
kwargs = dict(parse_qsl(parsed_url.query))
# Prepare the parameters to initialize imported class.
checkpoint_provider = parsed_url.netloc
KVStoreClass = import_attr(checkpoint_provider)
if not issubclass(KVStoreClass, KVStoreBase):
raise ValueError(
f"{KVStoreClass} doesn't inherit from "
"`ray.serve.storage.kv_store_base.KVStoreBase`.")
logger.info(
f"Using {checkpoint_provider} for controller checkpoint and "
f"recovery: kwargs={kwargs}")
return KVStoreClass(namespace=namespace, **kwargs)
raise RuntimeError("This shouldn't be reachable.")
def __init__(self, dashboard_agent):
super().__init__(dashboard_agent)
self._session_dir = dashboard_agent.session_dir
self._runtime_env_dir = dashboard_agent.runtime_env_dir
self._setup = import_attr(dashboard_agent.runtime_env_setup_hook)
runtime_env.PKG_DIR = dashboard_agent.runtime_env_dir
