def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None):
"""Returns a Syncer.
This syncer is in charge of syncing the local_dir with upload_dir.
Args:
local_dir (str): Source directory for syncing.
remote_dir (str): Target directory for syncing. If not provided, a
no-op Syncer is returned.
sync_function (func | str): Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults
to standard S3, gsutil or HDFS sync commands.
Raises:
ValueError if malformed remote_dir.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
if not remote_dir:
_syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP)
return _syncers[key]
client = get_sync_client(sync_function)
if client:
_syncers[key] = CloudSyncer(local_dir, remote_dir, client)
return _syncers[key]
sync_client = get_cloud_sync_client(remote_dir)
_syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def get_node_syncer(local_dir, remote_dir=None, sync_function=None):
"""Returns a NodeSyncer.
Args:
local_dir (str): Source directory for syncing.
remote_dir (str): Target directory for syncing. If not provided, a
noop Syncer is returned.
sync_function (func|str|bool): Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If True or not provided, it defaults rsync. If
False, a noop Syncer is returned.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
elif isclass(sync_function) and issubclass(sync_function, Syncer):
_syncers[key] = sync_function(local_dir, remote_dir, None)
return _syncers[key]
elif not remote_dir or sync_function is False:
sync_client = NOOP
elif sync_function and sync_function is not True:
sync_client = get_sync_client(sync_function)
else:
sync = log_sync_template()
if sync:
sync_client = CommandBasedClient(sync, sync)
sync_client.set_logdir(local_dir)
else:
sync_client = NOOP
_syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def get_node_syncer(local_dir, remote_dir=None, sync_function=None):
"""Returns a NodeSyncer.
Args:
local_dir (str): Source directory for syncing.
remote_dir (str): Target directory for syncing. If not provided, a
no-op Syncer is returned.
sync_function (func|str): Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults rsync.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
elif not remote_dir:
sync_client = NOOP
elif sync_function:
sync_client = get_sync_client(sync_function)
else:
sync_up = log_sync_template()
sync_down = log_sync_template(options="--remove-source-files")
if sync_up and sync_down:
sync_client = CommandBasedClient(sync_up, sync_down)
sync_client.set_logdir(local_dir)
else:
sync_client = NOOP
_syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def mock_storage_client():
"""Mocks storage client that treats a local dir as durable storage."""
client = get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE)
path = os.path.join(ray.utils.get_user_temp_dir(),
f"mock-client-{uuid.uuid4().hex[:4]}")
os.makedirs(path, exist_ok=True)
client.set_logdir(path)
return client
def get_cloud_syncer(
local_dir: str,
remote_dir: Optional[str] = None,
sync_function: Optional[Union[Callable, str]] = None,
) -> CloudSyncer:
"""Returns a Syncer.
This syncer is in charge of syncing the local_dir with upload_dir.
If no ``remote_dir`` is provided, it will return a no-op syncer.
If a ``sync_function`` is provided, it will return a CloudSyncer using
a custom SyncClient initialized by the sync function. Otherwise it will
return a CloudSyncer with default templates for s3/gs/hdfs.
Args:
local_dir: Source directory for syncing.
remote_dir: Target directory for syncing. If not provided, a
no-op Syncer is returned.
sync_function: Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults
to standard S3, gsutil or HDFS sync commands.
Raises:
ValueError if malformed remote_dir.
"""
key = (local_dir, remote_dir)
if key in _syncers:
return _syncers[key]
if not remote_dir:
_syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP)
return _syncers[key]
if sync_function == "auto":
sync_function = None # Auto-detect
# Maybe get user-provided sync client here
client = get_sync_client(sync_function)
if client:
# If the user provided a sync template or function
_syncers[key] = CloudSyncer(local_dir, remote_dir, client)
else:
# Else, get default cloud sync client (e.g. S3 syncer)
sync_client = get_cloud_sync_client(remote_dir)
_syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def get_node_syncer(
local_dir: str,
remote_dir: Optional[str] = None,
sync_function: Optional[Union[Callable, str, bool, Type[Syncer]]] = None,
):
"""Returns a NodeSyncer.
Args:
local_dir: Source directory for syncing.
remote_dir: Target directory for syncing. If not provided, a
noop Syncer is returned.
sync_function: Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If True or not provided, it defaults rsync
(if available) or otherwise remote-task based syncing. If
False, a noop Syncer is returned.
"""
if sync_function == "auto":
sync_function = None # Auto-detect
key = (local_dir, remote_dir)
if key in _syncers:
# Get cached syncer
return _syncers[key]
elif isclass(sync_function) and issubclass(sync_function, Syncer):
# Type[Syncer]
_syncers[key] = sync_function(local_dir, remote_dir, None)
return _syncers[key]
elif not remote_dir or sync_function is False:
# Do not sync trials if no remote dir specified or syncer=False
sync_client = NOOP
elif sync_function and sync_function is not True:
# String or callable (for function syncers)
sync_client = get_sync_client(sync_function)
else:
# sync_function=True or sync_function=None --> default
rsync_function_str = get_rsync_template_if_available()
if rsync_function_str:
sync_client = CommandBasedClient(rsync_function_str,
rsync_function_str)
sync_client.set_logdir(local_dir)
else:
sync_client = RemoteTaskClient()
_syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client)
return _syncers[key]
def _create_storage_client(self):
"""Returns a storage client."""
return get_sync_client(
self.sync_function_tpl) or get_cloud_sync_client(
self.remote_checkpoint_dir)
def mock_storage_client():
"""Mocks storage client that treats a local dir as durable storage."""
return get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE)
def _create_storage_client(self):
sync = "mkdir -p {target} && rsync -avz {source} {target}"
delete = "rm -rf {target}"
return get_sync_client(sync, delete)
def mock_storage_client(path):
"""Mocks storage client that treats a local dir as durable storage."""
client = get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE)
os.makedirs(path, exist_ok=True)
client.set_logdir(path)
return client
