def test_multipart_with_policy(self, mcg_obj, bucket_factory):
"""
Test Multipart upload with bucket policy set on the bucket
"""
bucket = bucket_factory(interface="OC")[0].name
obc_obj = OBC(bucket)
part_body = "Random data-" + str(uuid.uuid4().hex)
object_key = "MpuObjKey"
bucket_policy_generated = gen_bucket_policy(
user_list=obc_obj.obc_account,
actions_list=[
"ListBucketMultipartUploads",
"ListMultipartUploadParts",
"PutObject",
],
resources_list=[
obc_obj.bucket_name, f'{obc_obj.bucket_name}/{"*"}'
],
effect="Allow",
)
bucket_policy = json.dumps(bucket_policy_generated)
# Creates and gets policy
logger.info(f"Creating policy on bucket: {obc_obj.bucket_name}")
put_bucket_policy(mcg_obj, obc_obj.bucket_name, bucket_policy)
logger.info(f"Getting Bucket policy on bucket: {obc_obj.bucket_name}")
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
logger.info(
f"Initiating MP Upload on Bucket: {bucket} with Key {object_key}")
upload_id = create_multipart_upload(obc_obj, bucket, object_key)
logger.info(
f"Listing the MP Upload : {list_multipart_upload(obc_obj, bucket)['Uploads']}"
)
# Uploading individual part with no body to the Bucket
logger.info(f"Uploading to the bucket: {bucket}")
part_etag = obc_obj.s3_client.upload_part(
Bucket=bucket,
Key=object_key,
Body=part_body,
UploadId=upload_id,
PartNumber=1,
)["ETag"]
# Listing the Uploaded part
logger.info(
f"Listing the individual part: {list_uploaded_parts(obc_obj, bucket, object_key, upload_id)['Parts']}"
)
uploaded_part = [{"ETag": part_etag, "PartNumber": 1}]
# Completing the Multipart Upload
logger.info(f"Completing the MP Upload with on bucket: {bucket}")
complete_multipart_upload(obc_obj, bucket, object_key, upload_id,
uploaded_part)
def test_multipart_upload_operations(self, rgw_endpoint,
awscli_pod_session,
rgw_bucket_factory,
test_directory_setup):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod_session, rgw_bucket_factory, test_directory_setup)
bucketname = bucket.name
bucket = OBC(bucketname)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f"Aborting any Multipart Upload on bucket:{bucketname}")
abort_all_multipart_upload(bucket, bucketname, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f"Initiating Multipart Upload on Bucket: {bucketname} with Key {key}"
)
upload_id = create_multipart_upload(bucket, bucketname, key)
logger.info(
f"Listing the Multipart Upload: {list_multipart_upload(bucket, bucketname)}"
)
# Uploading individual parts to the Bucket
logger.info(f"Uploading individual parts to the bucket {bucketname}")
uploaded_parts = upload_parts(bucket, awscli_pod_session, bucketname,
key, res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f"Listing the individual parts: {list_uploaded_parts(bucket, bucketname, key, upload_id)}"
)
# Completing the Multipart Upload
logger.info(f"Completing the Multipart Upload on bucket: {bucketname}")
logger.info(
complete_multipart_upload(bucket, bucketname, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
"Downloading the completed multipart object from the RGW bucket to the awscli pod"
)
sync_object_directory(awscli_pod_session, object_path, res_dir, bucket)
assert verify_s3_object_integrity(
original_object_path=f"{origin_dir}/{key}",
result_object_path=f"{res_dir}/{key}",
awscli_pod=awscli_pod_session,
), "Checksum comparision between original and result object failed"
def test_multipart_with_no_body(self, mcg_obj, bucket_factory):
"""
Test Multipart upload with no body while uploading the part
"""
bucket = bucket_factory(amount=1, interface="OC")[0].name
object_key = "ObjKey-" + str(uuid.uuid4().hex)
# Create & list Multipart Upload on the Bucket
logger.info(
f"Initiating Multipart Upload on Bucket: {bucket} with Key {object_key}"
)
upload_id = create_multipart_upload(mcg_obj, bucket, object_key)
logger.info(
f"Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)['Uploads']}"
)
# Uploading individual part with no body to the Bucket
logger.info(
f"Uploading individual parts to the bucket: {bucket} with no body specified"
)
part_etag = mcg_obj.s3_client.upload_part(Bucket=bucket,
Key=object_key,
UploadId=upload_id,
PartNumber=1)["ETag"]
uploaded_part = [{"ETag": part_etag, "PartNumber": 1}]
# Listing the Uploaded part
logger.info(
f"Listing the individual part: {list_uploaded_parts(mcg_obj, bucket, object_key, upload_id)['Parts']}"
)
# Completing the Multipart Upload
logger.info(
f"Completing the Multipart Upload with a part and no body on bucket: {bucket}"
)
complete_multipart_upload(mcg_obj, bucket, object_key, upload_id,
uploaded_part)
def test_multipart_upload_operations(self, mcg_obj, awscli_pod,
bucket_factory):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod, bucket_factory)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f'Aborting any Multipart Upload on bucket:{bucket}')
abort_all_multipart_upload(mcg_obj, bucket, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f'Initiating Multipart Upload on Bucket: {bucket} with Key {key}')
upload_id = create_multipart_upload(mcg_obj, bucket, key)
logger.info(
f'Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)}'
)
# Uploading individual parts to the Bucket
logger.info(f'Uploading individual parts to the bucket {bucket}')
uploaded_parts = upload_parts(mcg_obj, awscli_pod, bucket, key,
res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f'Listing the individual parts : {list_uploaded_parts(mcg_obj, bucket, key, upload_id)}'
)
# Completing the Multipart Upload
logger.info(f'Completing the Multipart Upload on bucket: {bucket}')
logger.info(
complete_multipart_upload(mcg_obj, bucket, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
'Downloading the completed multipart object from MCG bucket to awscli pod'
)
sync_object_directory(awscli_pod, object_path, res_dir, mcg_obj)
assert verify_s3_object_integrity(
original_object_path=f'{origin_dir}/{key}',
result_object_path=f'{res_dir}/{key}',
awscli_pod=awscli_pod
), 'Checksum comparision between original and result object failed'
def test_mcg_namespace_mpu_crd(self, mcg_obj, awscli_pod, bucket_factory,
bucketclass_dict):
"""
Test multipart upload S3 operations on namespace buckets(created by CRDs)
Validates create, upload, upload copy and list parts operations
"""
ns_buc = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0]
ns_bucket = ns_buc.name
object_path = f"s3://{ns_bucket}"
logger.info(
f"Setting up test files for mpu and aborting any mpu on bucket: {ns_bucket}"
)
mpu_key, origin_dir, res_dir, parts = multipart_setup(awscli_pod)
bucket_utils.abort_all_multipart_upload(mcg_obj, ns_bucket, COPY_OBJ)
# Initiate mpu, Upload part copy, List and Abort operations
logger.info(
f"Put object on bucket: {ns_bucket} to create a copy source")
assert bucket_utils.s3_put_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ,
data=OBJ_DATA), "Failed: PutObject"
logger.info(
f"Initiating mpu on bucket: {ns_bucket} with key {COPY_OBJ}")
part_copy_id = bucket_utils.create_multipart_upload(
mcg_obj, ns_bucket, COPY_OBJ)
list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj,
bucketname=ns_bucket)
if (constants.AZURE_PLATFORM
not in bucketclass_dict["namespace_policy_dict"]
["namespacestore_dict"]):
logger.info(f"Listing in-progress mpu: {list_mpu_res}")
assert (part_copy_id == list_mpu_res["Uploads"][0]["UploadId"]
), "Invalid UploadId"
logger.info(f"Uploading a part copy to: {ns_bucket}")
assert bucket_utils.s3_upload_part_copy(
s3_obj=mcg_obj,
bucketname=ns_bucket,
copy_source=f"/{ns_bucket}/{ROOT_OBJ}",
object_key=COPY_OBJ,
part_number=1,
upload_id=part_copy_id,
), "Failed: upload part copy"
logger.info(
f"Aborting initiated multipart upload with id: {part_copy_id}")
assert bucket_utils.abort_multipart(mcg_obj, ns_bucket, COPY_OBJ,
part_copy_id), "Abort failed"
# Initiate mpu, Upload part, List parts operations
logger.info(
f"Initiating Multipart Upload on Bucket: {ns_bucket} with Key: {mpu_key}"
)
mp_upload_id = bucket_utils.create_multipart_upload(
mcg_obj, ns_bucket, mpu_key)
list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj,
bucketname=ns_bucket)
if (constants.AZURE_PLATFORM
not in bucketclass_dict["namespace_policy_dict"]
["namespacestore_dict"]):
logger.info(f"Listing multipart upload: {list_mpu_res}")
assert (mp_upload_id == list_mpu_res["Uploads"][0]["UploadId"]
), "Invalid UploadId"
logger.info(f"Uploading individual parts to the bucket: {ns_bucket}")
uploaded_parts = bucket_utils.upload_parts(
mcg_obj=mcg_obj,
awscli_pod=awscli_pod,
bucketname=ns_bucket,
object_key=mpu_key,
body_path=res_dir,
upload_id=mp_upload_id,
uploaded_parts=parts,
)
list_parts_res = bucket_utils.list_uploaded_parts(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=mpu_key,
upload_id=mp_upload_id,
)
logger.info(f"Listing individual parts: {list_parts_res['Parts']}")
for i, ele in enumerate(uploaded_parts):
assert (ele["PartNumber"] == list_parts_res["Parts"][i]
["PartNumber"]), "Invalid part_number"
assert ele["ETag"] == list_parts_res["Parts"][i][
"ETag"], "Invalid ETag"
logger.info(f"Completing the Multipart Upload on bucket: {ns_bucket}")
assert bucket_utils.complete_multipart_upload(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=mpu_key,
upload_id=mp_upload_id,
parts=uploaded_parts,
), "MPU did not complete"
# Checksum validation after completing MPU
logger.info(
f"Downloading the completed multipart object from {ns_bucket} to aws-cli pod"
)
bucket_utils.sync_object_directory(podobj=awscli_pod,
src=object_path,
target=res_dir,
s3_obj=mcg_obj)
assert bucket_utils.verify_s3_object_integrity(
original_object_path=f"{origin_dir}/{mpu_key}",
result_object_path=f"{res_dir}/{mpu_key}",
awscli_pod=awscli_pod,
), "Checksum comparision between original and result object failed"
