def _s3_to_s3_cp(from_path: str, to_path: str, overwrite: bool,
fs: s3fs.S3FileSystem, **kwargs) -> None:
from_path = _norm_s3_path(from_path)
to_path = _norm_s3_path(to_path)
files = fs.walk(from_path)
if files:
################################
# Copying a directory of files #
################################
to_files = [
os.path.join(to_path, f.replace(from_path + "/", ""))
for f in files
]
# Ensure we aren't overwriting any files
if not overwrite:
for to_file in to_files:
if already_exists(to_file, fs):
raise ValueError(
f"Overwrite set to False and {to_file!r} exists")
num_threads = kwargs.pop("num_threads", 100)
# Turn off connectionpool warnings
logging.getLogger("urllib3.connectionpool").setLevel(logging.CRITICAL)
with ThreadPoolExecutor(num_threads) as executor:
for from_file, to_file in zip(files, to_files):
executor.submit(fs.copy, from_file, to_file, **kwargs)
else:
#########################
# Copying a single file #
#########################
# Ensure we aren't overwriting the file
if not overwrite and already_exists(to_path, fs):
raise ValueError(f"Overwrite set to False and {to_file!r} exists")
fs.copy(from_path, to_path, **kwargs)
def copy(self, path1, path2, **kwargs):
return S3FileSystem.copy(self, get_key(path1), get_key(path2), **kwargs)
