def create(
self,
uri: str,
runtime_env: RuntimeEnv,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger,
) -> int:
logger.debug("Setting up pip for runtime_env: "
f"{runtime_env.serialize()}")
protocol, hash = parse_uri(uri)
target_dir = self._get_path_from_hash(hash)
pip_packages: List[str] = runtime_env.pip_packages()
with FileLock(self._installs_and_deletions_file_lock):
_install_pip_list_to_dir(pip_packages, target_dir, logger=logger)
# Despite Ray being removed from the input pip list during
# validation, other packages in the pip list (for example,
# xgboost_ray) may themselves include Ray as a dependency. In this
# case, we will have inadvertently installed the latest Ray version
# in the target_dir, which may cause Ray version mismatch issues.
# Uninstall it here, if it exists, to make the workers use the Ray
# that is already installed in the cluster.
#
# In the case where the user explicitly wants to include Ray in
# their pip list (and signals this by setting the environment
# variable below) then we don't want this deletion logic, so we
# skip it.
if os.environ.get(RAY_RUNTIME_ENV_ALLOW_RAY_IN_PIP) != 1:
ray_path = Path(target_dir) / "ray"
if ray_path.exists() and ray_path.is_dir():
shutil.rmtree(ray_path)
return get_directory_size_bytes(target_dir)
def setup(self,
runtime_env: RuntimeEnv,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger):
if not runtime_env.has_pip():
return
logger.debug(f"Setting up pip for runtime_env: {runtime_env}")
pip_packages: List[str] = runtime_env.pip_packages()
target_dir = self._get_path_from_hash(_get_pip_hash(pip_packages))
_install_pip_list_to_dir(pip_packages, target_dir, logger=logger)
# Despite Ray being removed from the input pip list during validation,
# other packages in the pip list (for example, xgboost_ray) may
# themselves include Ray as a dependency. In this case, we will have
# inadvertently installed the latest Ray version in the target_dir,
# which may cause Ray version mismatch issues. Uninstall it here, if it
# exists, to make the workers use the Ray that is already
# installed in the cluster.
#
# In the case where the user explicitly wants to include Ray in their
# pip list (and signals this by setting the environment variable below)
# then we don't want this deletion logic, so we skip it.
if os.environ.get(RAY_RUNTIME_ENV_ALLOW_RAY_IN_PIP) != 1:
ray_path = Path(target_dir) / "ray"
if ray_path.exists() and ray_path.is_dir():
shutil.rmtree(ray_path)
# Insert the target directory into the PYTHONPATH.
python_path = target_dir
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path