def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None): """Returns a Syncer. This syncer is in charge of syncing the local_dir with upload_dir. Args: local_dir (str): Source directory for syncing. remote_dir (str): Target directory for syncing. If not provided, a no-op Syncer is returned. sync_function (func | str): Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults to standard S3, gsutil or HDFS sync commands. Raises: ValueError if malformed remote_dir. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] if not remote_dir: _syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP) return _syncers[key] client = get_sync_client(sync_function) if client: _syncers[key] = CloudSyncer(local_dir, remote_dir, client) return _syncers[key] sync_client = get_cloud_sync_client(remote_dir) _syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def get_cloud_syncer( local_dir: str, remote_dir: Optional[str] = None, sync_function: Optional[Union[Callable, str]] = None, ) -> CloudSyncer: """Returns a Syncer. This syncer is in charge of syncing the local_dir with upload_dir. If no ``remote_dir`` is provided, it will return a no-op syncer. If a ``sync_function`` is provided, it will return a CloudSyncer using a custom SyncClient initialized by the sync function. Otherwise it will return a CloudSyncer with default templates for s3/gs/hdfs. Args: local_dir: Source directory for syncing. remote_dir: Target directory for syncing. If not provided, a no-op Syncer is returned. sync_function: Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults to standard S3, gsutil or HDFS sync commands. Raises: ValueError if malformed remote_dir. """ key = (local_dir, remote_dir) if key in _syncers: return _syncers[key] if not remote_dir: _syncers[key] = CloudSyncer(local_dir, remote_dir, NOOP) return _syncers[key] if sync_function == "auto": sync_function = None # Auto-detect # Maybe get user-provided sync client here client = get_sync_client(sync_function) if client: # If the user provided a sync template or function _syncers[key] = CloudSyncer(local_dir, remote_dir, client) else: # Else, get default cloud sync client (e.g. S3 syncer) sync_client = get_cloud_sync_client(remote_dir) _syncers[key] = CloudSyncer(local_dir, remote_dir, sync_client) return _syncers[key]
def _create_storage_client(self): """Returns a storage client.""" return get_sync_client( self.sync_function_tpl) or get_cloud_sync_client( self.remote_checkpoint_dir)