def test_exclude_file(self):
result = exclude_file(["*"], ["src/*"], "src/hello.txt")
self.assertEqual(result, False)
result = exclude_file(["*"], ["*"], "src/hello.txt")
self.assertEqual(result, False)
result = exclude_file(["*"], [".*"], "src/hello.txt")
self.assertEqual(result, True)
def recursive_upload(s3: S3, local_path: str, exclude: List[str],
include: List[str], extra_args: S3Args) -> None:
"""Recursive upload local directory to s3.
Perform a os.walk to upload everyfile under a directory.
:param s3: S3 instance
:type s3: S3
:param local_path: local directory
:type local_path: str
:param exclude: glob pattern to exclude
:type exclude: List[str]
:param include: glob pattern to include
:type include: List[str]
:param extra_args: S3Args instance to set extra argument
:type extra_args: S3Args
"""
upload_list: List[Dict[str, str]] = []
for root, _, files in os.walk(local_path):
for filename in files:
full_path = os.path.join(root, filename)
relative_path = os.path.relpath(full_path, local_path)
if not exclude_file(exclude, include, relative_path):
destination_key = s3.get_s3_destination_key(relative_path,
recursive=True)
print("(dryrun) upload: %s to s3://%s/%s" %
(relative_path, s3.bucket_name, destination_key))
upload_list.append({
"local_path": full_path,
"bucket": s3.bucket_name,
"key": destination_key,
"relative": relative_path,
})
if get_confirmation("Confirm?"):
for item in upload_list:
print("upload: %s to s3://%s/%s" %
(item["relative"], item["bucket"], item["key"]))
transfer = S3TransferWrapper(s3.client)
transfer.s3transfer.upload_file(
item["local_path"],
item["bucket"],
item["key"],
callback=S3Progress(item["local_path"]),
extra_args=extra_args.extra_args,
)
def find_all_version_files(
client,
bucket: str,
path: str,
file_list: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
include: Optional[List[str]] = None,
deletemark: bool = False,
) -> List[str]:
"""Find all files based on versions.
This method is able to find all files even deleted files or just delete marker left overs.
Use this method when needing to cleanly delete all files including their versions.
:param client: boto3 client
:type client: boto3.client
:param bucket: bucket to walk
:type bucket: str
:param path: the folder path to walk, empty to walk from root
:type path: str
:param file_list: list of files that was walked, for recursive purpose, pass empty list in
:type file_list: List[str], optional
:param exclude: glob pattern to exclude
:type exclude: List[str], optional
:param include: glob pattern to include
:type include: glob pattern to include
:param deletemark: set to true if only want to find deletemark
:type deletemark: bool, optional
:return: return a list of file names including deleted file names with delete mark remained
:rtype: List[str]
"""
if file_list is None:
file_list = []
if exclude is None:
exclude = []
if include is None:
include = []
paginator = client.get_paginator("list_object_versions")
for result in paginator.paginate(Bucket=bucket, Delimiter="/",
Prefix=path):
if result.get("CommonPrefixes") is not None:
for subdir in result.get("CommonPrefixes"):
file_list = find_all_version_files(client, bucket,
subdir.get("Prefix"),
file_list, exclude, include)
if not deletemark:
for file in result.get("Versions", []):
if exclude_file(exclude, include, file.get("Key")):
continue
if file.get("Key") in file_list:
continue
else:
file_list.append(file.get("Key"))
for file in result.get("DeleteMarkers", []):
if exclude_file(exclude, include, file.get("Key")):
continue
if file.get("Key") in file_list:
continue
else:
file_list.append(file.get("Key"))
return file_list
def test_exclude_file_exclude(self):
result = exclude_file([".git/*"], [], ".git/COMMIT_EDITMSG")
self.assertEqual(result, True)
result = exclude_file([".git/*"], [], "gitconfig.json")
self.assertEqual(result, False)
def test_exclude_file_include(self):
result = exclude_file([], [".git/*"], ".git/COMMIT_EDITMSG")
self.assertEqual(result, False)
result = exclude_file([], [".git/*"], "hello.txt")
self.assertEqual(result, False)