def to_uri(self, uri: str) -> str:
"""Write checkpoint data to location URI (e.g. cloud storage).
ARgs:
uri (str): Target location URI to write data to.
Returns:
str: Cloud location containing checkpoint data.
"""
if uri.startswith("file://"):
local_path = uri[7:]
return self.to_directory(local_path)
assert is_cloud_target(uri)
cleanup = False
local_path = self._local_path
if not local_path:
cleanup = True
local_path = self.to_directory()
upload_to_bucket(bucket=uri, local_path=local_path)
if cleanup:
shutil.rmtree(local_path)
return uri
def upload(
self,
cloud_path: Optional[str] = None,
local_path: Optional[str] = None,
clean_before: bool = False,
):
"""Upload checkpoint to cloud.
This will push the checkpoint directory from local storage
to ``cloud_path``.
If a ``cloud_path`` argument is provided and ``self.cloud_path``
is unset, it will be set to ``cloud_path``.
Args:
cloud_path (Optional[str]): Cloud path to load checkpoint from.
Defaults to ``self.cloud_path``.
local_path (Optional[str]): Local path to save checkpoint at.
Defaults to ``self.local_path``.
clean_before (bool): If True, deletes potentially existing
cloud bucket before storing new data.
"""
local_path = local_path or self.local_path
if not local_path:
raise RuntimeError(
"Could not upload trial checkpoint: No local "
"path is set. Fix this by either passing a "
"`local_path` to your call to `upload()` or by "
"passing a `local_path` into the constructor."
)
cloud_path = cloud_path or self.cloud_path
if not cloud_path:
raise RuntimeError(
"Could not download trial checkpoint: No cloud "
"path is set. Fix this by either passing a "
"`cloud_path` to your call to `download()` or by "
"passing a `cloud_path` into the constructor. The latter "
"should automatically be done if you pass the correct "
"`tune.SyncConfig`."
)
if not self.cloud_path:
self.cloud_path = cloud_path
if clean_before:
logger.info(f"Clearing bucket contents before upload: {cloud_path}")
clear_bucket(cloud_path)
# Actually upload
upload_to_bucket(cloud_path, local_path)
return cloud_path