def _s3_to_s3_cp(from_path: str, to_path: str, overwrite: bool, fs: s3fs.S3FileSystem, **kwargs) -> None: from_path = _norm_s3_path(from_path) to_path = _norm_s3_path(to_path) files = fs.walk(from_path) if files: ################################ # Copying a directory of files # ################################ to_files = [ os.path.join(to_path, f.replace(from_path + "/", "")) for f in files ] # Ensure we aren't overwriting any files if not overwrite: for to_file in to_files: if already_exists(to_file, fs): raise ValueError( f"Overwrite set to False and {to_file!r} exists") num_threads = kwargs.pop("num_threads", 100) # Turn off connectionpool warnings logging.getLogger("urllib3.connectionpool").setLevel(logging.CRITICAL) with ThreadPoolExecutor(num_threads) as executor: for from_file, to_file in zip(files, to_files): executor.submit(fs.copy, from_file, to_file, **kwargs) else: ######################### # Copying a single file # ######################### # Ensure we aren't overwriting the file if not overwrite and already_exists(to_path, fs): raise ValueError(f"Overwrite set to False and {to_file!r} exists") fs.copy(from_path, to_path, **kwargs)
def copy(self, path1, path2, **kwargs): return S3FileSystem.copy(self, get_key(path1), get_key(path2), **kwargs)