def _get_local_path(path: Optional[str]) -> Optional[str]:
"""Check if path is a local path. Otherwise return None."""
if path is None or is_non_local_path_uri(path):
return None
if path.startswith("file://"):
path = path[7:]
if os.path.exists(path):
return path
return None
def _validate_upload_dir(sync_config: "SyncConfig") -> bool:
if not sync_config.upload_dir:
return True
if sync_config.upload_dir.startswith("file://"):
return True
if not is_non_local_path_uri(sync_config.upload_dir):
raise ValueError(f"Could not identify external storage filesystem for "
f"upload dir `{sync_config.upload_dir}`. "
f"Hint: {fs_hint(sync_config.upload_dir)}")
def to_uri(self, uri: str) -> str:
"""Write checkpoint data to location URI (e.g. cloud storage).
Args:
uri: Target location URI to write data to.
Returns:
str: Cloud location containing checkpoint data.
"""
if uri.startswith("file://"):
local_path = uri[7:]
return self.to_directory(local_path)
if not is_non_local_path_uri(uri):
raise RuntimeError(
f"Cannot upload checkpoint to URI: Provided URI "
f"does not belong to a registered storage provider: `{uri}`. "
f"Hint: {fs_hint(uri)}")
with self.as_directory() as local_path:
upload_to_uri(local_path=local_path, uri=uri)
return uri
def _get_external_path(path: Optional[str]) -> Optional[str]:
"""Check if path is an external path. Otherwise return None."""
if not isinstance(path, str) or not is_non_local_path_uri(path):
return None
return path
