def test_check_object_integrity(self, mcg_obj, awscli_pod, bucket_factory):
"""
Test object integrity using md5sum
"""
bucketname = bucket_factory(1)[0].name
original_dir = "/original"
result_dir = "/result"
awscli_pod.exec_cmd_on_pod(command=f'mkdir {result_dir}')
# Retrieve a list of all objects on the test-objects bucket and
# downloads them to the pod
full_object_path = f"s3://{bucketname}"
downloaded_files = retrieve_test_objects_to_pod(
awscli_pod, original_dir)
# Write all downloaded objects to the new bucket
sync_object_directory(awscli_pod, original_dir, full_object_path,
mcg_obj)
# Retrieve all objects from MCG bucket to result dir in Pod
logger.info('Downloading all objects from MCG bucket to awscli pod')
sync_object_directory(awscli_pod, full_object_path, result_dir,
mcg_obj)
# Checksum is compared between original and result object
for obj in downloaded_files:
assert verify_s3_object_integrity(
original_object_path=f'{original_dir}/{obj}',
result_object_path=f'{result_dir}/{obj}',
awscli_pod=awscli_pod
), 'Checksum comparision between original and result object failed'
def test_check_object_integrity(self, rgw_endpoint, awscli_pod_session,
rgw_bucket_factory, test_directory_setup):
"""
Test object integrity using md5sum
"""
bucketname = rgw_bucket_factory(1, "rgw-oc")[0].name
obc_obj = OBC(bucketname)
original_dir = AWSCLI_TEST_OBJ_DIR
result_dir = test_directory_setup.result_dir
full_object_path = f"s3://{bucketname}"
downloaded_files = awscli_pod_session.exec_cmd_on_pod(
f"ls -A1 {original_dir}").split(" ")
# Write all downloaded objects to the new bucket
sync_object_directory(awscli_pod_session, original_dir,
full_object_path, obc_obj)
logger.info("Downloading all objects from RGW bucket to awscli pod")
sync_object_directory(awscli_pod_session, full_object_path, result_dir,
obc_obj)
# Checksum is compared between original and result object
for obj in downloaded_files:
assert verify_s3_object_integrity(
original_object_path=f"{original_dir}/{obj}",
result_object_path=f"{result_dir}/{obj}",
awscli_pod=awscli_pod_session,
), "Checksum comparision between original and result object failed"
def test_write_file_to_bucket(
self,
mcg_obj,
awscli_pod,
bucket_class_factory,
bucket_factory,
interface,
bucketclass_dict,
):
"""
Test object IO using the S3 SDK
"""
# Retrieve a list of all objects on the test-objects bucket and
# downloads them to the pod
data_dir = "/data"
bucketname = bucket_factory(1,
interface=interface,
bucketclass=bucketclass_dict)[0].name
full_object_path = f"s3://{bucketname}"
downloaded_files = retrieve_test_objects_to_pod(awscli_pod, data_dir)
# Write all downloaded objects to the new bucket
sync_object_directory(awscli_pod, data_dir, full_object_path, mcg_obj)
assert set(downloaded_files).issubset(
obj.key
for obj in mcg_obj.s3_list_all_objects_in_bucket(bucketname))
def test_bucket_delete_with_objects(self, mcg_obj, interface, awscli_pod):
"""
Negative test with deletion of bucket has objects stored in.
"""
bucketname = create_unique_resource_name(
resource_description="bucket", resource_type=interface.lower())
try:
bucket = BUCKET_MAP[interface.lower()](bucketname, mcg=mcg_obj)
logger.info(f"aws s3 endpoint is {mcg_obj.s3_endpoint}")
logger.info(f"aws region is {mcg_obj.region}")
data_dir = "/data"
full_object_path = f"s3://{bucketname}"
retrieve_test_objects_to_pod(awscli_pod, data_dir)
sync_object_directory(awscli_pod, data_dir, full_object_path,
mcg_obj)
logger.info(f"Deleting bucket: {bucketname}")
if interface == "S3":
try:
s3_del = mcg_obj.s3_resource.Bucket(bucketname).delete()
assert not s3_del, "Unexpected s3 delete non-empty OBC succeed"
except botocore.exceptions.ClientError as err:
assert "BucketNotEmpty" in str(
err), "Couldn't verify delete non-empty OBC with s3"
logger.info(
f"Delete non-empty OBC {bucketname} failed as expected"
)
finally:
bucket.delete()
def test_empty_file_integrity(self, awscli_pod, rgw_bucket_factory):
"""
Test write empty files to bucket and check integrity
"""
original_dir = "/data"
result_dir = "/result"
bucketname = rgw_bucket_factory(1, "rgw-oc")[0].name
obc_obj = OBC(bucketname)
full_object_path = f"s3://{bucketname}"
# Touch create 1000 empty files in pod
awscli_pod.exec_cmd_on_pod(command=f"mkdir {original_dir} {result_dir}")
command = "for i in $(seq 1 100); do touch /data/test$i; done"
awscli_pod.exec_sh_cmd_on_pod(command=command, sh="sh")
# Write all empty objects to the new bucket
sync_object_directory(awscli_pod, original_dir, full_object_path, obc_obj)
# Retrieve all objects from RGW bucket to result dir in Pod
logger.info("Downloading objects from RGW bucket to awscli pod")
sync_object_directory(awscli_pod, full_object_path, result_dir, obc_obj)
# Checksum is compared between original and result object
original_md5 = awscli_pod.exec_cmd_on_pod(
f'sh -c "cat {original_dir}/* | md5sum"'
)
result_md5 = awscli_pod.exec_cmd_on_pod(
f'sh -c "cat {original_dir}/* | md5sum"'
)
assert (
original_md5 == result_md5
), "Origin and result folders checksum mismatch found"
def write_files_to_pod_and_upload(
self,
mcg_obj,
awscli_pod,
bucket_to_write,
original_folder,
amount=1,
s3_creds=None,
):
"""
Upload files to bucket (NS or uls)
"""
full_object_path = f"s3://{bucket_to_write}"
object_list = []
for i in range(amount):
file_name = f"testfile{i}.txt"
object_list.append(file_name)
awscli_pod.exec_cmd_on_pod(
f"dd if=/dev/urandom of={original_folder}/{file_name} bs=1M count=1 status=none"
)
if s3_creds:
# Write data directly to target bucket from original dir
sync_object_directory(
awscli_pod,
original_folder,
full_object_path,
signed_request_creds=s3_creds,
)
else:
# Write data directly to NS bucket from original dir
sync_object_directory(
awscli_pod, original_folder, full_object_path, mcg_obj
)
return object_list
def test_s3_bucket_delete_1t_objects(self, mcg_obj, awscli_pod_session):
"""
Test with deletion of bucket has 1T objects stored in.
"""
bucketname = create_unique_resource_name(resource_description="bucket",
resource_type="s3")
try:
bucket = MCGS3Bucket(bucketname, mcg_obj)
logger.info(f"aws s3 endpoint is {mcg_obj.s3_endpoint}")
logger.info(f"aws region is {mcg_obj.region}")
data_dir = AWSCLI_TEST_OBJ_DIR
# Sync downloaded objects dir to the new bucket, sync to 3175
# virtual dirs. With each dir around 315MB, and 3175 dirs will
# reach targed 1TB data.
logger.info("Writing objects to bucket")
for i in range(3175):
full_object_path = f"s3://{bucketname}/{i}/"
sync_object_directory(awscli_pod_session, data_dir,
full_object_path, mcg_obj)
# Delete bucket content use aws rm with --recursive option.
# The object_versions.delete function does not work with objects
# exceeds 1000.
start = timeit.default_timer()
rm_object_recursive(awscli_pod_session, bucketname, mcg_obj)
bucket.delete()
stop = timeit.default_timer()
gap = (stop - start) // 60 % 60
if gap > 10:
assert False, "Failed to delete s3 bucket within 10 minutes"
finally:
if mcg_obj.s3_verify_bucket_exists(bucketname):
rm_object_recursive(awscli_pod_session, bucketname, mcg_obj)
mcg_obj.s3_resource.Bucket(bucketname).delete()
def write_files_to_pod_and_upload(self,
mcg_obj,
awscli_pod,
bucket_to_write,
amount=1,
s3_creds=None):
"""
Upload files to bucket (NS or uls)
"""
awscli_pod.exec_cmd_on_pod(
command=f"mkdir -p {self.MCG_NS_ORIGINAL_DIR}")
full_object_path = f"s3://{bucket_to_write}"
for i in range(amount):
file_name = f"testfile{i}"
awscli_pod.exec_cmd_on_pod(
f"dd if=/dev/urandom of={self.MCG_NS_ORIGINAL_DIR}/{file_name}.txt bs=1M count=1 status=none"
)
if s3_creds:
# Write data directly to target bucket from original dir
sync_object_directory(
awscli_pod,
self.MCG_NS_ORIGINAL_DIR,
full_object_path,
signed_request_creds=s3_creds,
)
else:
# Write data directly to NS bucket from original dir
sync_object_directory(awscli_pod, self.MCG_NS_ORIGINAL_DIR,
full_object_path, mcg_obj)
def test_check_object_integrity(self, awscli_pod, rgw_bucket_factory):
"""
Test object integrity using md5sum
"""
bucketname = rgw_bucket_factory(1, "rgw-oc")[0].name
obc_obj = OBC(bucketname)
original_dir = "/original"
result_dir = "/result"
awscli_pod.exec_cmd_on_pod(command=f"mkdir {result_dir}")
# Retrieve a list of all objects on the test-objects bucket and
# downloads them to the pod
full_object_path = f"s3://{bucketname}"
downloaded_files = retrieve_test_objects_to_pod(awscli_pod, original_dir)
# Write all downloaded objects to the new bucket
sync_object_directory(awscli_pod, original_dir, full_object_path, obc_obj)
logger.info("Downloading all objects from RGW bucket to awscli pod")
sync_object_directory(awscli_pod, full_object_path, result_dir, obc_obj)
# Checksum is compared between original and result object
for obj in downloaded_files:
assert verify_s3_object_integrity(
original_object_path=f"{original_dir}/{obj}",
result_object_path=f"{result_dir}/{obj}",
awscli_pod=awscli_pod,
), "Checksum comparision between original and result object failed"
def test_write_file_to_bucket(
self,
mcg_obj,
awscli_pod_session,
bucket_class_factory,
bucket_factory,
interface,
bucketclass_dict,
):
"""
Test object IO using the S3 SDK
"""
# Retrieve a list of all objects on the test-objects bucket and
# downloads them to the pod
bucketname = bucket_factory(
1, interface=interface, bucketclass=bucketclass_dict
)[0].name
full_object_path = f"s3://{bucketname}"
downloaded_files = awscli_pod_session.exec_cmd_on_pod(
f"ls -A1 {AWSCLI_TEST_OBJ_DIR}"
).split(" ")
# Write all downloaded objects to the new bucket
sync_object_directory(
awscli_pod_session, AWSCLI_TEST_OBJ_DIR, full_object_path, mcg_obj
)
assert set(downloaded_files).issubset(
obj.key for obj in mcg_obj.s3_list_all_objects_in_bucket(bucketname)
)
def test_bucket_delete_with_objects(self, rgw_endpoint, rgw_bucket_factory,
interface, awscli_pod_session):
"""
Negative test with deletion of bucket has objects stored in.
"""
bucket = rgw_bucket_factory(1, interface)[0]
bucketname = bucket.name
obc_obj = OBC(bucketname)
try:
data_dir = AWSCLI_TEST_OBJ_DIR
full_object_path = f"s3://{bucketname}"
sync_object_directory(awscli_pod_session, data_dir,
full_object_path, obc_obj)
logger.info(f"Deleting bucket: {bucketname}")
if interface == "S3":
try:
s3_del = obc_obj.s3_resource.Bucket(bucketname).delete()
assert (
not s3_del
), "Unexpected issue: Successfully deleted a bucket containing objects via S3"
except botocore.exceptions.ClientError as err:
assert "BucketNotEmpty" in str(
err), "Couldn't verify delete non-empty OBC with s3"
logger.info(
f"Delete non-empty OBC {bucketname} failed as expected"
)
finally:
bucket.delete()
def test_bucket_delete_with_objects(
self, mcg_obj, awscli_pod, bucket_factory, interface, bucketclass_dict
):
"""
Negative test with deletion of bucket has objects stored in.
"""
bucket = bucket_factory(interface=interface, bucketclass=bucketclass_dict)[0]
bucketname = bucket.name
data_dir = "/data"
full_object_path = f"s3://{bucketname}"
retrieve_test_objects_to_pod(awscli_pod, data_dir)
sync_object_directory(awscli_pod, data_dir, full_object_path, mcg_obj)
logger.info(f"Deleting bucket: {bucketname}")
if interface == "S3":
try:
s3_del = mcg_obj.s3_resource.Bucket(bucketname).delete()
assert not s3_del, "Unexpected s3 delete non-empty OBC succeed"
except botocore.exceptions.ClientError as err:
assert "BucketNotEmpty" in str(
err
), "Couldn't verify delete non-empty OBC with s3"
logger.info(f"Delete non-empty OBC {bucketname} failed as expected")
# Deletion verification is performed internally as part of delete()
bucket.delete()
if bucketclass_dict:
bucket.bucketclass.delete()
def test_check_object_integrity(
self,
mcg_obj,
awscli_pod,
bucket_factory,
bucketclass_dict,
test_directory_setup,
):
"""
Test object integrity using md5sum
"""
bucketname = bucket_factory(1, bucketclass=bucketclass_dict)[0].name
original_dir = test_directory_setup.origin_dir
result_dir = test_directory_setup.result_dir
# Retrieve a list of all objects on the test-objects bucket and
# downloads them to the pod
full_object_path = f"s3://{bucketname}"
downloaded_files = retrieve_test_objects_to_pod(awscli_pod, original_dir)
# Write all downloaded objects to the new bucket
sync_object_directory(awscli_pod, original_dir, full_object_path, mcg_obj)
# Retrieve all objects from MCG bucket to result dir in Pod
logger.info("Downloading all objects from MCG bucket to awscli pod")
sync_object_directory(awscli_pod, full_object_path, result_dir, mcg_obj)
# Checksum is compared between original and result object
for obj in downloaded_files:
assert verify_s3_object_integrity(
original_object_path=f"{original_dir}/{obj}",
result_object_path=f"{result_dir}/{obj}",
awscli_pod=awscli_pod,
), "Checksum comparison between original and result object failed"
def test_empty_file_integrity(self, mcg_obj, awscli_pod, bucket_factory):
"""
Test write empty files to bucket and check integrity
"""
original_dir = '/data'
result_dir = "/result"
bucketname = bucket_factory(1)[0].name
full_object_path = f"s3://{bucketname}"
# Touch create 1000 empty files in pod
awscli_pod.exec_cmd_on_pod(
command=f'mkdir {original_dir} {result_dir}')
command = "for i in $(seq 1 100); do touch /data/test$i; done"
awscli_pod.exec_sh_cmd_on_pod(command=command, sh='sh')
# Write all empty objects to the new bucket
sync_object_directory(awscli_pod, original_dir, full_object_path,
mcg_obj)
# Retrieve all objects from MCG bucket to result dir in Pod
logger.info('Downloading objects from MCG bucket to awscli pod')
sync_object_directory(awscli_pod, full_object_path, result_dir,
mcg_obj)
# Checksum is compared between original and result object
original_md5 = awscli_pod.exec_cmd_on_pod(
f'sh -c "cat {original_dir}/* | md5sum"')
result_md5 = awscli_pod.exec_cmd_on_pod(
f'sh -c "cat {original_dir}/* | md5sum"')
assert original_md5 == result_md5
def test_check_multi_object_integrity(self, mcg_obj, awscli_pod,
bucket_factory, amount, file_type):
"""
Test write multiple files to bucket and check integrity
"""
original_dir = "/original"
result_dir = "/result"
if file_type == 'large':
public_bucket = PUBLIC_BUCKET
obj_key = LARGE_FILE_KEY
elif file_type == 'small':
public_bucket = constants.TEST_FILES_BUCKET
obj_key = 'random1.txt'
elif file_type == 'large_small':
public_bucket = PUBLIC_BUCKET
obj_key = LARGE_FILE_KEY.rsplit('/', 1)[0]
# Download the file to pod
awscli_pod.exec_cmd_on_pod(
command=f'mkdir {original_dir} {result_dir}')
public_s3_client = retrieve_anon_s3_resource().meta.client
download_files = []
# Use obj_key as prefix to download multiple files for large_small
# case, it also works with single file
for obj in public_s3_client.list_objects(
Bucket=public_bucket, Prefix=obj_key).get('Contents'):
# Skip the extra file in large file type
if file_type == 'large' and obj["Key"] != obj_key:
continue
logger.info(
f'Downloading {obj["Key"]} from AWS bucket {public_bucket}')
download_obj_cmd = f'cp s3://{public_bucket}/{obj["Key"]} {original_dir}'
awscli_pod.exec_cmd_on_pod(
command=craft_s3_command(download_obj_cmd),
out_yaml_format=False)
download_files.append(obj['Key'].split('/')[-1])
# Write downloaded objects to the new bucket and check integrity
bucketname = bucket_factory(1)[0].name
base_path = f"s3://{bucketname}"
for i in range(amount):
full_object_path = base_path + f"/{i}/"
sync_object_directory(awscli_pod, original_dir, full_object_path,
mcg_obj)
# Retrieve all objects from MCG bucket to result dir in Pod
logger.info('Downloading objects from MCG bucket to awscli pod')
sync_object_directory(awscli_pod, full_object_path, result_dir,
mcg_obj)
# Checksum is compared between original and result object
for obj in download_files:
assert verify_s3_object_integrity(
original_object_path=f'{original_dir}/{obj}',
result_object_path=f'{result_dir}/{obj}',
awscli_pod=awscli_pod
), ('Checksum comparision between original and result object '
'failed')
def test_write_multi_files_to_bucket(self, mcg_obj, awscli_pod,
bucket_factory, amount, file_type):
"""
Test write multiple files to bucket
"""
data_dir = "/data"
if file_type == "large":
public_bucket = PUBLIC_BUCKET
obj_key = LARGE_FILE_KEY
elif file_type == "small":
public_bucket = constants.TEST_FILES_BUCKET
obj_key = "random1.txt"
elif file_type == "large_small":
public_bucket = PUBLIC_BUCKET
obj_key = LARGE_FILE_KEY.rsplit("/", 1)[0]
# Download the file to pod
awscli_pod.exec_cmd_on_pod(command=f"mkdir {data_dir}")
public_s3_client = retrieve_anon_s3_resource().meta.client
download_files = []
# Use obj_key as prefix to download multiple files for large_small
# case, it also works with single file
for obj in public_s3_client.list_objects(
Bucket=public_bucket, Prefix=obj_key).get("Contents"):
# Skip the extra file in large file type
if file_type == "large" and obj["Key"] != obj_key:
continue
logger.info(
f'Downloading {obj["Key"]} from AWS bucket {public_bucket}')
download_obj_cmd = f'cp s3://{public_bucket}/{obj["Key"]} {data_dir}'
awscli_pod.exec_cmd_on_pod(
command=craft_s3_command(download_obj_cmd),
out_yaml_format=False)
download_files.append(obj["Key"])
# Write all downloaded objects to the new bucket
bucketname = bucket_factory(1)[0].name
base_path = f"s3://{bucketname}"
for i in range(amount):
full_object_path = base_path + f"/{i}/"
sync_object_directory(awscli_pod, data_dir, full_object_path,
mcg_obj)
obj_list = list(
obj.key.split("/")[-1]
for obj in mcg_obj.s3_list_all_objects_in_bucket(bucketname))
# Check total copy files amount match
if file_type == "large_small":
assert len(
obj_list) == 2 * amount, "Total file amount does not match"
else:
assert len(obj_list) == amount, "Total file amount does not match"
# Check deduplicate set is same
test_set = set([i.split("/")[-1] for i in download_files])
assert test_set == set(obj_list), "File name set does not match"
def test_multipart_upload_operations(self, rgw_endpoint,
awscli_pod_session,
rgw_bucket_factory,
test_directory_setup):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod_session, rgw_bucket_factory, test_directory_setup)
bucketname = bucket.name
bucket = OBC(bucketname)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f"Aborting any Multipart Upload on bucket:{bucketname}")
abort_all_multipart_upload(bucket, bucketname, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f"Initiating Multipart Upload on Bucket: {bucketname} with Key {key}"
)
upload_id = create_multipart_upload(bucket, bucketname, key)
logger.info(
f"Listing the Multipart Upload: {list_multipart_upload(bucket, bucketname)}"
)
# Uploading individual parts to the Bucket
logger.info(f"Uploading individual parts to the bucket {bucketname}")
uploaded_parts = upload_parts(bucket, awscli_pod_session, bucketname,
key, res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f"Listing the individual parts: {list_uploaded_parts(bucket, bucketname, key, upload_id)}"
)
# Completing the Multipart Upload
logger.info(f"Completing the Multipart Upload on bucket: {bucketname}")
logger.info(
complete_multipart_upload(bucket, bucketname, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
"Downloading the completed multipart object from the RGW bucket to the awscli pod"
)
sync_object_directory(awscli_pod_session, object_path, res_dir, bucket)
assert verify_s3_object_integrity(
original_object_path=f"{origin_dir}/{key}",
result_object_path=f"{res_dir}/{key}",
awscli_pod=awscli_pod_session,
), "Checksum comparision between original and result object failed"
def test_data_reduction_performance(self, mcg_obj, awscli_pod, bucket_factory):
"""
Test data reduction performance
"""
# TODO: Privatize test bucket
download_dir = "/aws/downloaded"
retrieve_test_objects_to_pod(awscli_pod, download_dir)
bucket = bucket_factory(1)[0]
bucketname = bucket.name
full_object_path = f"s3://{bucketname}"
sync_object_directory(awscli_pod, download_dir, full_object_path, mcg_obj)
assert mcg_obj.check_data_reduction(
bucketname, 100 * 1024 * 1024
), "Data reduction did not work as anticipated."
def test_fill_bucket(
mcg_obj_session, awscli_pod_session, multiregion_mirror_setup_session
):
"""
Test multi-region bucket creation using the S3 SDK. Fill the bucket for
upgrade testing.
"""
(bucket, created_backingstores) = multiregion_mirror_setup_session
mcg_bucket_path = f"s3://{bucket.name}"
# Download test objects from the public bucket
awscli_pod_session.exec_cmd_on_pod(command=f"mkdir {LOCAL_TESTOBJS_DIR_PATH}")
DOWNLOADED_OBJS = retrieve_test_objects_to_pod(
awscli_pod_session, LOCAL_TESTOBJS_DIR_PATH
)
logger.info("Uploading all pod objects to MCG bucket")
# Upload test objects to the NooBucket 3 times
for i in range(3):
sync_object_directory(
awscli_pod_session,
LOCAL_TESTOBJS_DIR_PATH,
f"{mcg_bucket_path}/{i}/",
mcg_obj_session,
)
mcg_obj_session.check_if_mirroring_is_done(bucket.name, timeout=420)
bucket.verify_health()
# Retrieve all objects from MCG bucket to result dir in Pod
sync_object_directory(
awscli_pod_session, mcg_bucket_path, LOCAL_TEMP_PATH, mcg_obj_session
)
bucket.verify_health()
# Checksum is compared between original and result object
for obj in DOWNLOADED_OBJS:
for i in range(3):
assert verify_s3_object_integrity(
original_object_path=f"{LOCAL_TESTOBJS_DIR_PATH}/{obj}",
result_object_path=f"{LOCAL_TEMP_PATH}/{i}/{obj}",
awscli_pod=awscli_pod_session,
), "Checksum comparison between original and result object failed"
bucket.verify_health()
def test_data_reduction(self, mcg_obj, awscli_pod, bucket_factory):
"""
Test data reduction mechanics
"""
# TODO: Privatize test bucket
download_dir = '/aws/downloaded'
retrieve_test_objects_to_pod(awscli_pod, download_dir)
bucket = bucket_factory(1)[0]
bucketname = bucket.name
full_object_path = f"s3://{bucketname}"
sync_object_directory(awscli_pod, download_dir, full_object_path,
mcg_obj)
assert mcg_obj.check_data_reduction(bucketname), (
'Data reduction did not work as anticipated.')
def test_multipart_upload_operations(self, mcg_obj, awscli_pod,
bucket_factory):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod, bucket_factory)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f'Aborting any Multipart Upload on bucket:{bucket}')
abort_all_multipart_upload(mcg_obj, bucket, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f'Initiating Multipart Upload on Bucket: {bucket} with Key {key}')
upload_id = create_multipart_upload(mcg_obj, bucket, key)
logger.info(
f'Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)}'
)
# Uploading individual parts to the Bucket
logger.info(f'Uploading individual parts to the bucket {bucket}')
uploaded_parts = upload_parts(mcg_obj, awscli_pod, bucket, key,
res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f'Listing the individual parts : {list_uploaded_parts(mcg_obj, bucket, key, upload_id)}'
)
# Completing the Multipart Upload
logger.info(f'Completing the Multipart Upload on bucket: {bucket}')
logger.info(
complete_multipart_upload(mcg_obj, bucket, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
'Downloading the completed multipart object from MCG bucket to awscli pod'
)
sync_object_directory(awscli_pod, object_path, res_dir, mcg_obj)
assert verify_s3_object_integrity(
original_object_path=f'{origin_dir}/{key}',
result_object_path=f'{res_dir}/{key}',
awscli_pod=awscli_pod
), 'Checksum comparision between original and result object failed'
def download_files(
self, mcg_obj, awscli_pod, bucket_to_read, result_folder, s3_creds=None
):
"""
Download files from bucket (NS or uls)
"""
ns_bucket_path = f"s3://{bucket_to_read}"
if s3_creds:
# Read data directly from target bucket (uls) to result dir
sync_object_directory(
awscli_pod,
ns_bucket_path,
result_folder,
signed_request_creds=s3_creds,
)
else:
# Read data from NS bucket to result dir
sync_object_directory(awscli_pod, ns_bucket_path, result_folder, mcg_obj)
def test_fill_bucket(mcg_obj_session, awscli_pod_session,
multiregion_mirror_setup_session):
"""
Test multi-region bucket creation using the S3 SDK. Fill the bucket for
upgrade testing.
"""
(bucket, created_backingstores) = multiregion_mirror_setup_session
mcg_bucket_path = f's3://{bucket.name}'
# Download test objects from the public bucket
awscli_pod_session.exec_cmd_on_pod(
command=f'mkdir {LOCAL_TESTOBJS_DIR_PATH}')
DOWNLOADED_OBJS = retrieve_test_objects_to_pod(awscli_pod_session,
LOCAL_TESTOBJS_DIR_PATH)
logger.info('Uploading all pod objects to MCG bucket')
# Upload test objects to the NooBucket 3 times
for i in range(3):
sync_object_directory(awscli_pod_session, LOCAL_TESTOBJS_DIR_PATH,
f'{mcg_bucket_path}/{i}/', mcg_obj_session)
mcg_obj_session.check_if_mirroring_is_done(bucket.name)
assert bucket.status == constants.STATUS_BOUND
# Retrieve all objects from MCG bucket to result dir in Pod
sync_object_directory(awscli_pod_session, mcg_bucket_path, LOCAL_TEMP_PATH,
mcg_obj_session)
assert bucket.status == constants.STATUS_BOUND
# Checksum is compared between original and result object
for obj in DOWNLOADED_OBJS:
for i in range(3):
assert verify_s3_object_integrity(
original_object_path=f'{LOCAL_TESTOBJS_DIR_PATH}/{obj}',
result_object_path=f'{LOCAL_TEMP_PATH}/{i}/{obj}',
awscli_pod=awscli_pod_session
), 'Checksum comparison between original and result object failed'
assert bucket.status == constants.STATUS_BOUND
def download_files(self, mcg_obj, awscli_pod, bucket_to_read, s3_creds=None):
"""
Download files from bucket (NS or uls)
"""
awscli_pod.exec_cmd_on_pod(command=f"mkdir {self.MCG_NS_RESULT_DIR}")
ns_bucket_path = f"s3://{bucket_to_read}"
if s3_creds:
# Read data directly from target bucket (uls) to result dir
sync_object_directory(
awscli_pod,
ns_bucket_path,
self.MCG_NS_RESULT_DIR,
signed_request_creds=s3_creds,
)
else:
# Read data from NS bucket to result dir
sync_object_directory(
awscli_pod, ns_bucket_path, self.MCG_NS_RESULT_DIR, mcg_obj
)
def test_write_empty_file_to_bucket(self, mcg_obj, awscli_pod, bucket_factory):
"""
Test write empty files to bucket
"""
data_dir = "/data"
bucketname = bucket_factory(1)[0].name
full_object_path = f"s3://{bucketname}"
# Touch create 1000 empty files in pod
awscli_pod.exec_cmd_on_pod(command=f"mkdir {data_dir}")
command = "for i in $(seq 1 100); do touch /data/test$i; done"
awscli_pod.exec_sh_cmd_on_pod(command=command, sh="sh")
# Write all empty objects to the new bucket
sync_object_directory(awscli_pod, data_dir, full_object_path, mcg_obj)
obj_set = set(
obj.key for obj in mcg_obj.s3_list_all_objects_in_bucket(bucketname)
)
test_set = set("test" + str(i + 1) for i in range(100))
assert test_set == obj_set, "File name set does not match"
def test_noobaa_postupgrade(mcg_obj_session, awscli_pod_session,
multiregion_mirror_setup_session):
"""
Check bucket data and remove resources created in 'test_fill_bucket'.
"""
(bucket, created_backingstores) = multiregion_mirror_setup_session
backingstore1 = created_backingstores[0]
backingstore2 = created_backingstores[1]
mcg_bucket_path = f"s3://{bucket.name}"
# Checksum is compared between original and result object
for obj in DOWNLOADED_OBJS:
assert verify_s3_object_integrity(
original_object_path=f"{LOCAL_TESTOBJS_DIR_PATH}/{obj}",
result_object_path=f"{LOCAL_TEMP_PATH}/{obj}",
awscli_pod=awscli_pod_session,
), "Checksum comparision between original and result object failed"
bucket.verify_health()
# Clean up the temp dir
awscli_pod_session.exec_cmd_on_pod(
command=f'sh -c "rm -rf {LOCAL_TEMP_PATH}/*"')
mcg_obj_session.check_backingstore_state("backing-store-" +
backingstore1.name,
BS_OPTIMAL,
timeout=360)
mcg_obj_session.check_backingstore_state("backing-store-" +
backingstore2.name,
BS_OPTIMAL,
timeout=360)
bucket.verify_health()
# Verify integrity of A
# Retrieve all objects from MCG bucket to result dir in Pod
sync_object_directory(awscli_pod_session, mcg_bucket_path, LOCAL_TEMP_PATH,
mcg_obj_session)
bucket.verify_health()
def test_noobaa_postupgrade(mcg_obj_session, awscli_pod_session,
multiregion_mirror_setup_session):
"""
Check bucket data and remove resources created in 'test_fill_bucket'.
"""
(bucket, created_backingstores) = multiregion_mirror_setup_session
backingstore1 = created_backingstores[0]
backingstore2 = created_backingstores[1]
mcg_bucket_path = f's3://{bucket.name}'
# Checksum is compared between original and result object
for obj in DOWNLOADED_OBJS:
assert verify_s3_object_integrity(
original_object_path=f'{LOCAL_TESTOBJS_DIR_PATH}/{obj}',
result_object_path=f'{LOCAL_TEMP_PATH}/{obj}',
awscli_pod=awscli_pod_session
), 'Checksum comparision between original and result object failed'
assert bucket.status == constants.STATUS_BOUND
# Clean up the temp dir
awscli_pod_session.exec_cmd_on_pod(
command=f'sh -c \"rm -rf {LOCAL_TEMP_PATH}/*\"')
mcg_obj_session.check_backingstore_state('backing-store-' +
backingstore1['name'],
BS_OPTIMAL,
timeout=360)
mcg_obj_session.check_backingstore_state('backing-store-' +
backingstore2['name'],
BS_OPTIMAL,
timeout=360)
assert bucket.status == constants.STATUS_BOUND
# Verify integrity of A
# Retrieve all objects from MCG bucket to result dir in Pod
sync_object_directory(awscli_pod_session, mcg_bucket_path, LOCAL_TEMP_PATH,
mcg_obj_session)
assert bucket.status == constants.STATUS_BOUND
def write_empty_files_to_bucket(mcg_obj, awscli_pod_session, bucket_name,
test_directory_setup):
"""
Write empty files to bucket and verify if they are created.
Args:
mcg_obj (MCG) : An MCG object containing the MCG S3 connection credentials
awscli_pod_session : Fixture to create a new AWSCLI pod for relaying commands.
bucket_name (str) : Name of the bucket on which files are to be written.
test_directory_setup : Fixture to setup test DIRs.
Raises:
UnexpectedBehaviour : Raises an exception if files are not created.
Returns:
Set: A set of names of all bucket objects.
"""
full_object_path = f"s3://{bucket_name}"
data_dir = test_directory_setup.origin_dir
# Touch create 1000 empty files in bucket
command = f"for file_no in $(seq 1 1000); do touch {data_dir}/test$file_no; done"
awscli_pod_session.exec_sh_cmd_on_pod(command=command, sh="sh")
# Write all empty objects to the bucket
sync_object_directory(awscli_pod_session, data_dir, full_object_path,
mcg_obj)
log.info("Successfully created files.")
obj_set = set(
obj.key for obj in mcg_obj.s3_list_all_objects_in_bucket(bucket_name))
test_set = set("test" + str(file_no + 1) for file_no in range(1000))
if test_set != obj_set:
raise ex.UnexpectedBehaviour("File name set does not match")
log.info("File name set match")
return obj_set
def test_mcg_data_compression(self, mcg_obj, awscli_pod, bucket_factory,
bucketclass_dict):
"""
Test data reduction mechanics
Args:
mcg_obj (obj): An object representing the current state of the MCG in the cluster
awscli_pod (pod): A pod running the AWSCLI tools
bucket_factory: Calling this fixture creates a new bucket(s)
"""
download_dir = "/aws/compression/"
awscli_pod.exec_cmd_on_pod(
command=craft_s3_command(
f"cp s3://{constants.TEST_FILES_BUCKET}/enwik8 {download_dir}"
),
out_yaml_format=False,
)
bucketname = bucket_factory(1, bucketclass=bucketclass_dict)[0].name
full_object_path = f"s3://{bucketname}"
sync_object_directory(awscli_pod, download_dir, full_object_path,
mcg_obj)
# For this test, enwik8 is used in conjunction with Snappy compression
# utilized by NooBaa. Snappy consistently compresses 35MB of the file.
mcg_obj.check_data_reduction(bucketname, 35 * 1024 * 1024)
def test_bucket_delete_with_objects(self, mcg_obj, awscli_pod_session,
bucket_factory, interface,
bucketclass_dict):
"""
Negative test with deletion of bucket has objects stored in.
"""
bucketname = bucket_factory(bucketclass=bucketclass_dict)[0].name
data_dir = AWSCLI_TEST_OBJ_DIR
full_object_path = f"s3://{bucketname}"
sync_object_directory(awscli_pod_session, data_dir, full_object_path,
mcg_obj)
logger.info(f"Deleting bucket: {bucketname}")
if interface == "S3":
try:
s3_del = mcg_obj.s3_resource.Bucket(bucketname).delete()
assert not s3_del, "Unexpected s3 delete non-empty OBC succeed"
except botocore.exceptions.ClientError as err:
assert "BucketNotEmpty" in str(
err), "Couldn't verify delete non-empty OBC with s3"
logger.info(
f"Delete non-empty OBC {bucketname} failed as expected")
