def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None):
"""Returns a Syncer.
This syncer is in charge of syncing the local_dir with upload_dir.
Args:
local_dir (str): Source directory for syncing.
remote_dir (str): Target directory for syncing. If not provided, a
no-op Syncer is returned.
sync_function (func | str): Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults
to standard S3, gsutil or HDFS sync commands.
Raises:
ValueError if malformed remote_dir.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
if not remote_dir:
_syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP)
return _syncers[key]
client = get_sync_client(sync_function)
if client:
_syncers[key] = CloudSyncer(local_dir, remote_dir, client)
return _syncers[key]
sync_client = get_cloud_sync_client(remote_dir)
_syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def get_cloud_syncer(
local_dir: str,
remote_dir: Optional[str] = None,
sync_function: Optional[Union[Callable, str]] = None,
) -> CloudSyncer:
"""Returns a Syncer.
This syncer is in charge of syncing the local_dir with upload_dir.
If no ``remote_dir`` is provided, it will return a no-op syncer.
If a ``sync_function`` is provided, it will return a CloudSyncer using
a custom SyncClient initialized by the sync function. Otherwise it will
return a CloudSyncer with default templates for s3/gs/hdfs.
Args:
local_dir: Source directory for syncing.
remote_dir: Target directory for syncing. If not provided, a
no-op Syncer is returned.
sync_function: Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults
to standard S3, gsutil or HDFS sync commands.
Raises:
ValueError if malformed remote_dir.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
if not remote_dir:
_syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP)
return _syncers[key]
if sync_function == "auto":
sync_function = None # Auto-detect
# Maybe get user-provided sync client here
client = get_sync_client(sync_function)
if client:
# If the user provided a sync template or function
_syncers[key] = CloudSyncer(local_dir, remote_dir, client)
else:
# Else, get default cloud sync client (e.g. S3 syncer)
sync_client = get_cloud_sync_client(remote_dir)
_syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def _create_storage_client(self):
"""Returns a storage client."""
return get_sync_client(
self.sync_function_tpl) or get_cloud_sync_client(
self.remote_checkpoint_dir)
