def test_s3_local_cp_file(self, s3_fs, tmpdir):
from_path = f"s3://{test_bucket_name}/foo/bar.txt"
to_path = tmpdir.join("foobar.txt")
s3.cp(from_path, to_path, fs=s3_fs)
assert to_path.read() == "This is test file bar"
def test_local_s3_cp_dir(self, sample_local_dir, s3_fs):
from_path = os.path.join(sample_local_dir, "foo")
to_path = f"s3://{test_bucket_name}/tmp/test2"
bar_path = os.path.join(to_path, "foo", "bar.txt")
buzz_path = os.path.join(to_path, "foo", "fizz", "buzz.txt")
with pytest.raises(FileNotFoundError):
s3_fs.cat(buzz_path)
s3.cp(from_path, to_path, fs=s3_fs)
assert s3_fs.cat(bar_path) == b"This is test file bar"
assert s3_fs.cat(buzz_path) == b"This is test file buzz"
# Copy contents of directory
from_path = os.path.join(sample_local_dir, "foo")
to_path = f"s3://{test_bucket_name}/tmp/test2"
bar_path = os.path.join(to_path, "bar.txt")
buzz_path = os.path.join(to_path, "fizz", "buzz.txt")
with pytest.raises(FileNotFoundError):
s3_fs.cat(buzz_path)
s3.cp(from_path, to_path, include_folder_name=False, fs=s3_fs)
assert s3_fs.cat(bar_path) == b"This is test file bar"
assert s3_fs.cat(buzz_path) == b"This is test file buzz"
def test_local_s3_cp_file(self, sample_local_dir, s3_fs):
from_path = os.path.join(sample_local_dir, "foo", "bar.txt")
to_path = f"s3://{test_bucket_name}/tmp/test/bar.txt"
with pytest.raises(FileNotFoundError):
s3_fs.cat(to_path)
s3.cp(from_path, to_path, fs=s3_fs)
assert s3_fs.cat(to_path) == b"This is test file bar"
def test_s3_s3_cp_file(self, s3_fs):
from_path = f"s3://{test_bucket_name}/foo/bar.txt"
to_path = f"s3://{test_bucket_name}/tmp3/bar.txt"
with pytest.raises(FileNotFoundError):
s3_fs.cat(to_path)
s3.cp(from_path, to_path, fs=s3_fs)
assert s3_fs.cat(from_path) == b"This is test file bar"
assert s3_fs.cat(to_path) == b"This is test file bar"
def test_s3_local_cp_dir(self, s3_fs, tmpdir):
from_path = f"s3://{test_bucket_name}/foo"
bar_path = tmpdir.join("foo/bar.txt")
buzz_path = tmpdir.join("foo/fizz/buzz.txt")
s3.cp(from_path, tmpdir, fs=s3_fs)
assert bar_path.read() == "This is test file bar"
assert buzz_path.read() == "This is test file buzz"
# Copy contents of folder
bar_path = tmpdir.join("bar.txt")
buzz_path = tmpdir.join("fizz/buzz.txt")
s3.cp(from_path, tmpdir, include_folder_name=False, fs=s3_fs)
assert bar_path.read() == "This is test file bar"
assert buzz_path.read() == "This is test file buzz"
def test_s3_s3_cp_dir(self, s3_fs):
from_path = f"s3://{test_bucket_name}/foo"
to_path = f"s3://{test_bucket_name}/tmp4"
bar_path = f"s3://{test_bucket_name}/tmp4/foo/bar.txt"
with pytest.raises(FileNotFoundError):
s3_fs.cat(bar_path)
s3.cp(from_path, to_path, fs=s3_fs)
assert s3_fs.cat(bar_path) == b"This is test file bar"
# Copy the contents of folder
from_path = f"s3://{test_bucket_name}/foo"
to_path = f"s3://{test_bucket_name}/tmp5"
bar_path = f"s3://{test_bucket_name}/tmp5/bar.txt"
with pytest.raises(FileNotFoundError):
s3_fs.cat(bar_path)
s3.cp(from_path, to_path, include_folder_name=False, fs=s3_fs)
assert s3_fs.cat(bar_path) == b"This is test file bar"
def cp(from_path: str,
to_path: str,
overwrite: bool = True,
include_folder_name: bool = True,
**kwargs) -> None:
""" Copy a file or directory of files from local/s3 to local/s3
Parameters
-----------
from_path : str
Directory/file path to copy
to_path : str
Path to copy file(s) to.
overwrite : bool (default True)
Should the to_path be overwritten if it already exists?
include_folder_name : bool (default True)
If copying a directory, add the directory name automatically to the
to_path. i.e. if True, the entire folder will be copied to the
to_path. If False, the *contents* of the directory will be copied to
the to_path
kwargs : Dict
Extra arguments to pass to the appropriate cp (either _local.cp or
_s3.cp)
Returns
--------
None
"""
if s3.is_s3path(from_path) or s3.is_s3path(to_path):
s3.cp(from_path, to_path, overwrite, include_folder_name, **kwargs)
else:
local.cp(from_path, to_path, overwrite, include_folder_name)