def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None): """Returns a Syncer. This syncer is in charge of syncing the local_dir with upload_dir. Args: local_dir (str): Source directory for syncing. remote_dir (str): Target directory for syncing. If not provided, a no-op Syncer is returned. sync_function (func | str): Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults to standard S3, gsutil or HDFS sync commands. Raises: ValueError if malformed remote_dir. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] if not remote_dir: _syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP) return _syncers[key] client = get_sync_client(sync_function) if client: _syncers[key] = CloudSyncer(local_dir, remote_dir, client) return _syncers[key] sync_client = get_cloud_sync_client(remote_dir) _syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def get_node_syncer(local_dir, remote_dir=None, sync_function=None): """Returns a NodeSyncer. Args: local_dir (str): Source directory for syncing. remote_dir (str): Target directory for syncing. If not provided, a noop Syncer is returned. sync_function (func|str|bool): Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If True or not provided, it defaults rsync. If False, a noop Syncer is returned. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] elif isclass(sync_function) and issubclass(sync_function, Syncer): _syncers[key] = sync_function(local_dir, remote_dir, None) return _syncers[key] elif not remote_dir or sync_function is False: sync_client = NOOP elif sync_function and sync_function is not True: sync_client = get_sync_client(sync_function) else: sync = log_sync_template() if sync: sync_client = CommandBasedClient(sync, sync) sync_client.set_logdir(local_dir) else: sync_client = NOOP _syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def get_node_syncer(local_dir, remote_dir=None, sync_function=None): """Returns a NodeSyncer. Args: local_dir (str): Source directory for syncing. remote_dir (str): Target directory for syncing. If not provided, a no-op Syncer is returned. sync_function (func|str): Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults rsync. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] elif not remote_dir: sync_client = NOOP elif sync_function: sync_client = get_sync_client(sync_function) else: sync_up = log_sync_template() sync_down = log_sync_template(options="--remove-source-files") if sync_up and sync_down: sync_client = CommandBasedClient(sync_up, sync_down) sync_client.set_logdir(local_dir) else: sync_client = NOOP _syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def mock_storage_client(): """Mocks storage client that treats a local dir as durable storage.""" client = get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE) path = os.path.join(ray.utils.get_user_temp_dir(), f"mock-client-{uuid.uuid4().hex[:4]}") os.makedirs(path, exist_ok=True) client.set_logdir(path) return client
def get_cloud_syncer( local_dir: str, remote_dir: Optional[str] = None, sync_function: Optional[Union[Callable, str]] = None, ) -> CloudSyncer: """Returns a Syncer. This syncer is in charge of syncing the local_dir with upload_dir. If no ``remote_dir`` is provided, it will return a no-op syncer. If a ``sync_function`` is provided, it will return a CloudSyncer using a custom SyncClient initialized by the sync function. Otherwise it will return a CloudSyncer with default templates for s3/gs/hdfs. Args: local_dir: Source directory for syncing. remote_dir: Target directory for syncing. If not provided, a no-op Syncer is returned. sync_function: Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults to standard S3, gsutil or HDFS sync commands. Raises: ValueError if malformed remote_dir. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] if not remote_dir: _syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP) return _syncers[key] if sync_function == "auto": sync_function = None # Auto-detect # Maybe get user-provided sync client here client = get_sync_client(sync_function) if client: # If the user provided a sync template or function _syncers[key] = CloudSyncer(local_dir, remote_dir, client) else: # Else, get default cloud sync client (e.g. S3 syncer) sync_client = get_cloud_sync_client(remote_dir) _syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def get_node_syncer( local_dir: str, remote_dir: Optional[str] = None, sync_function: Optional[Union[Callable, str, bool, Type[Syncer]]] = None, ): """Returns a NodeSyncer. Args: local_dir: Source directory for syncing. remote_dir: Target directory for syncing. If not provided, a noop Syncer is returned. sync_function: Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If True or not provided, it defaults rsync (if available) or otherwise remote-task based syncing. If False, a noop Syncer is returned. """ if sync_function == "auto": sync_function = None # Auto-detect key = (local_dir, remote_dir) if key in _syncers: # Get cached syncer return _syncers[key] elif isclass(sync_function) and issubclass(sync_function, Syncer): # Type[Syncer] _syncers[key] = sync_function(local_dir, remote_dir, None) return _syncers[key] elif not remote_dir or sync_function is False: # Do not sync trials if no remote dir specified or syncer=False sync_client = NOOP elif sync_function and sync_function is not True: # String or callable (for function syncers) sync_client = get_sync_client(sync_function) else: # sync_function=True or sync_function=None --> default rsync_function_str = get_rsync_template_if_available() if rsync_function_str: sync_client = CommandBasedClient(rsync_function_str, rsync_function_str) sync_client.set_logdir(local_dir) else: sync_client = RemoteTaskClient() _syncers[key] = NodeSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def _create_storage_client(self): """Returns a storage client.""" return get_sync_client( self.sync_function_tpl) or get_cloud_sync_client( self.remote_checkpoint_dir)
def mock_storage_client(): """Mocks storage client that treats a local dir as durable storage.""" return get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE)
def _create_storage_client(self): sync = "mkdir -p {target} && rsync -avz {source} {target}" delete = "rm -rf {target}" return get_sync_client(sync, delete)
def mock_storage_client(path): """Mocks storage client that treats a local dir as durable storage.""" client = get_sync_client(LOCAL_SYNC_TEMPLATE, LOCAL_DELETE_TEMPLATE) os.makedirs(path, exist_ok=True) client.set_logdir(path) return client