def delete_uri(self,
uri: str,
logger: Optional[logging.Logger] = default_logger) -> int:
"""Delete URI and return the number of bytes deleted."""
local_dir = get_local_dir_from_uri(uri, self._resources_dir)
local_dir_size = get_directory_size_bytes(local_dir)
deleted = delete_package(uri, self._resources_dir)
if not deleted:
logger.warning(f"Tried to delete nonexistent URI: {uri}.")
return 0
return local_dir_size
def modify_context(self, uri: Optional[str], runtime_env_dict: Dict,
context: RuntimeEnvContext):
if uri is None:
return
local_dir = get_local_dir_from_uri(uri, self._resources_dir)
if not local_dir.exists():
raise ValueError(
f"Local directory {local_dir} for URI {uri} does "
"not exist on the cluster. Something may have gone wrong while "
"downloading or unpacking the working_dir.")
context.command_prefix += [f"cd {local_dir}"]
set_pythonpath_in_context(python_path=str(local_dir), context=context)
def modify_context(self, uri: Optional[str], runtime_env_dict: Dict,
context: RuntimeEnvContext):
if uri is None:
return
local_dir = get_local_dir_from_uri(uri, self._resources_dir)
if not local_dir.exists():
raise ValueError(
f"Local directory {local_dir} for URI {uri} does "
"not exist on the cluster. Something may have gone wrong while "
"downloading or unpacking the working_dir.")
context.command_prefix += [f"cd {local_dir}"]
# Insert the working_dir as the first entry in PYTHONPATH. This is
# compatible with users providing their own PYTHONPATH in env_vars.
python_path = str(local_dir)
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path
def modify_context(
self,
uri: str,
runtime_env: RuntimeEnv,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger,
):
if not runtime_env.has_pip():
return
# Insert the target directory into the PYTHONPATH.
protocol, hash = parse_uri(uri)
target_dir = get_local_dir_from_uri(uri, self._resources_dir)
if not target_dir.exists():
raise ValueError(
f"Local directory {target_dir} for URI {uri} does "
"not exist on the cluster. Something may have gone wrong while "
"installing the runtime_env `pip` packages.")
python_path = str(target_dir)
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path
def modify_context(
self,
uris: Optional[List[str]],
runtime_env_dict: Dict,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger,
):
if uris is None:
return
module_dirs = []
for uri in uris:
module_dir = get_local_dir_from_uri(uri, self._resources_dir)
if not module_dir.exists():
raise ValueError(
f"Local directory {module_dir} for URI {uri} does "
"not exist on the cluster. Something may have gone wrong while "
"downloading or unpacking the py_modules files.")
module_dirs.append(str(module_dir))
# Insert the py_modules directories into the PYTHONPATH.
python_path = os.pathsep.join(module_dirs)
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path
def test_get_local_dir_from_uri():
uri = "gcs://<working_dir_content_hash>.zip"
assert get_local_dir_from_uri(
uri, "base_dir") == Path("base_dir/<working_dir_content_hash>")
def _get_local_dir_from_uri(self, uri: str):
return get_local_dir_from_uri(uri, self._resources_dir)
