def delete_uri(self, uri: str, logger: Optional[logging.Logger] = default_logger) -> int: """Delete URI and return the number of bytes deleted.""" local_dir = get_local_dir_from_uri(uri, self._resources_dir) local_dir_size = get_directory_size_bytes(local_dir) deleted = delete_package(uri, self._resources_dir) if not deleted: logger.warning(f"Tried to delete nonexistent URI: {uri}.") return 0 return local_dir_size
def modify_context(self, uri: Optional[str], runtime_env_dict: Dict, context: RuntimeEnvContext): if uri is None: return local_dir = get_local_dir_from_uri(uri, self._resources_dir) if not local_dir.exists(): raise ValueError( f"Local directory {local_dir} for URI {uri} does " "not exist on the cluster. Something may have gone wrong while " "downloading or unpacking the working_dir.") context.command_prefix += [f"cd {local_dir}"] set_pythonpath_in_context(python_path=str(local_dir), context=context)
def modify_context(self, uri: Optional[str], runtime_env_dict: Dict, context: RuntimeEnvContext): if uri is None: return local_dir = get_local_dir_from_uri(uri, self._resources_dir) if not local_dir.exists(): raise ValueError( f"Local directory {local_dir} for URI {uri} does " "not exist on the cluster. Something may have gone wrong while " "downloading or unpacking the working_dir.") context.command_prefix += [f"cd {local_dir}"] # Insert the working_dir as the first entry in PYTHONPATH. This is # compatible with users providing their own PYTHONPATH in env_vars. python_path = str(local_dir) if "PYTHONPATH" in context.env_vars: python_path += os.pathsep + context.env_vars["PYTHONPATH"] context.env_vars["PYTHONPATH"] = python_path
def modify_context( self, uri: str, runtime_env: RuntimeEnv, context: RuntimeEnvContext, logger: Optional[logging.Logger] = default_logger, ): if not runtime_env.has_pip(): return # Insert the target directory into the PYTHONPATH. protocol, hash = parse_uri(uri) target_dir = get_local_dir_from_uri(uri, self._resources_dir) if not target_dir.exists(): raise ValueError( f"Local directory {target_dir} for URI {uri} does " "not exist on the cluster. Something may have gone wrong while " "installing the runtime_env `pip` packages.") python_path = str(target_dir) if "PYTHONPATH" in context.env_vars: python_path += os.pathsep + context.env_vars["PYTHONPATH"] context.env_vars["PYTHONPATH"] = python_path
def modify_context( self, uris: Optional[List[str]], runtime_env_dict: Dict, context: RuntimeEnvContext, logger: Optional[logging.Logger] = default_logger, ): if uris is None: return module_dirs = [] for uri in uris: module_dir = get_local_dir_from_uri(uri, self._resources_dir) if not module_dir.exists(): raise ValueError( f"Local directory {module_dir} for URI {uri} does " "not exist on the cluster. Something may have gone wrong while " "downloading or unpacking the py_modules files.") module_dirs.append(str(module_dir)) # Insert the py_modules directories into the PYTHONPATH. python_path = os.pathsep.join(module_dirs) if "PYTHONPATH" in context.env_vars: python_path += os.pathsep + context.env_vars["PYTHONPATH"] context.env_vars["PYTHONPATH"] = python_path
def test_get_local_dir_from_uri(): uri = "gcs://<working_dir_content_hash>.zip" assert get_local_dir_from_uri( uri, "base_dir") == Path("base_dir/<working_dir_content_hash>")
def _get_local_dir_from_uri(self, uri: str): return get_local_dir_from_uri(uri, self._resources_dir)