def test_multipart_upload_operations(self, rgw_endpoint, awscli_pod_session, rgw_bucket_factory, test_directory_setup): """ Test Multipart upload operations on bucket and verifies the integrity of the downloaded object """ bucket, key, origin_dir, res_dir, object_path, parts = setup( awscli_pod_session, rgw_bucket_factory, test_directory_setup) bucketname = bucket.name bucket = OBC(bucketname) # Abort all Multipart Uploads for this Bucket (optional, for starting over) logger.info(f"Aborting any Multipart Upload on bucket:{bucketname}") abort_all_multipart_upload(bucket, bucketname, key) # Create & list Multipart Upload on the Bucket logger.info( f"Initiating Multipart Upload on Bucket: {bucketname} with Key {key}" ) upload_id = create_multipart_upload(bucket, bucketname, key) logger.info( f"Listing the Multipart Upload: {list_multipart_upload(bucket, bucketname)}" ) # Uploading individual parts to the Bucket logger.info(f"Uploading individual parts to the bucket {bucketname}") uploaded_parts = upload_parts(bucket, awscli_pod_session, bucketname, key, res_dir, upload_id, parts) # Listing the Uploaded parts logger.info( f"Listing the individual parts: {list_uploaded_parts(bucket, bucketname, key, upload_id)}" ) # Completing the Multipart Upload logger.info(f"Completing the Multipart Upload on bucket: {bucketname}") logger.info( complete_multipart_upload(bucket, bucketname, key, upload_id, uploaded_parts)) # Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity logger.info( "Downloading the completed multipart object from the RGW bucket to the awscli pod" ) sync_object_directory(awscli_pod_session, object_path, res_dir, bucket) assert verify_s3_object_integrity( original_object_path=f"{origin_dir}/{key}", result_object_path=f"{res_dir}/{key}", awscli_pod=awscli_pod_session, ), "Checksum comparision between original and result object failed"
def test_multipart_upload_operations(self, mcg_obj, awscli_pod, bucket_factory): """ Test Multipart upload operations on bucket and verifies the integrity of the downloaded object """ bucket, key, origin_dir, res_dir, object_path, parts = setup( awscli_pod, bucket_factory) # Abort all Multipart Uploads for this Bucket (optional, for starting over) logger.info(f'Aborting any Multipart Upload on bucket:{bucket}') abort_all_multipart_upload(mcg_obj, bucket, key) # Create & list Multipart Upload on the Bucket logger.info( f'Initiating Multipart Upload on Bucket: {bucket} with Key {key}') upload_id = create_multipart_upload(mcg_obj, bucket, key) logger.info( f'Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)}' ) # Uploading individual parts to the Bucket logger.info(f'Uploading individual parts to the bucket {bucket}') uploaded_parts = upload_parts(mcg_obj, awscli_pod, bucket, key, res_dir, upload_id, parts) # Listing the Uploaded parts logger.info( f'Listing the individual parts : {list_uploaded_parts(mcg_obj, bucket, key, upload_id)}' ) # Completing the Multipart Upload logger.info(f'Completing the Multipart Upload on bucket: {bucket}') logger.info( complete_multipart_upload(mcg_obj, bucket, key, upload_id, uploaded_parts)) # Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity logger.info( 'Downloading the completed multipart object from MCG bucket to awscli pod' ) sync_object_directory(awscli_pod, object_path, res_dir, mcg_obj) assert verify_s3_object_integrity( original_object_path=f'{origin_dir}/{key}', result_object_path=f'{res_dir}/{key}', awscli_pod=awscli_pod ), 'Checksum comparision between original and result object failed'
def test_mcg_namespace_mpu_crd(self, mcg_obj, awscli_pod, bucket_factory, bucketclass_dict): """ Test multipart upload S3 operations on namespace buckets(created by CRDs) Validates create, upload, upload copy and list parts operations """ ns_buc = bucket_factory( amount=1, interface=bucketclass_dict["interface"], bucketclass=bucketclass_dict, )[0] ns_bucket = ns_buc.name object_path = f"s3://{ns_bucket}" logger.info( f"Setting up test files for mpu and aborting any mpu on bucket: {ns_bucket}" ) mpu_key, origin_dir, res_dir, parts = multipart_setup(awscli_pod) bucket_utils.abort_all_multipart_upload(mcg_obj, ns_bucket, COPY_OBJ) # Initiate mpu, Upload part copy, List and Abort operations logger.info( f"Put object on bucket: {ns_bucket} to create a copy source") assert bucket_utils.s3_put_object(s3_obj=mcg_obj, bucketname=ns_bucket, object_key=ROOT_OBJ, data=OBJ_DATA), "Failed: PutObject" logger.info( f"Initiating mpu on bucket: {ns_bucket} with key {COPY_OBJ}") part_copy_id = bucket_utils.create_multipart_upload( mcg_obj, ns_bucket, COPY_OBJ) list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj, bucketname=ns_bucket) if (constants.AZURE_PLATFORM not in bucketclass_dict["namespace_policy_dict"] ["namespacestore_dict"]): logger.info(f"Listing in-progress mpu: {list_mpu_res}") assert (part_copy_id == list_mpu_res["Uploads"][0]["UploadId"] ), "Invalid UploadId" logger.info(f"Uploading a part copy to: {ns_bucket}") assert bucket_utils.s3_upload_part_copy( s3_obj=mcg_obj, bucketname=ns_bucket, copy_source=f"/{ns_bucket}/{ROOT_OBJ}", object_key=COPY_OBJ, part_number=1, upload_id=part_copy_id, ), "Failed: upload part copy" logger.info( f"Aborting initiated multipart upload with id: {part_copy_id}") assert bucket_utils.abort_multipart(mcg_obj, ns_bucket, COPY_OBJ, part_copy_id), "Abort failed" # Initiate mpu, Upload part, List parts operations logger.info( f"Initiating Multipart Upload on Bucket: {ns_bucket} with Key: {mpu_key}" ) mp_upload_id = bucket_utils.create_multipart_upload( mcg_obj, ns_bucket, mpu_key) list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj, bucketname=ns_bucket) if (constants.AZURE_PLATFORM not in bucketclass_dict["namespace_policy_dict"] ["namespacestore_dict"]): logger.info(f"Listing multipart upload: {list_mpu_res}") assert (mp_upload_id == list_mpu_res["Uploads"][0]["UploadId"] ), "Invalid UploadId" logger.info(f"Uploading individual parts to the bucket: {ns_bucket}") uploaded_parts = bucket_utils.upload_parts( mcg_obj=mcg_obj, awscli_pod=awscli_pod, bucketname=ns_bucket, object_key=mpu_key, body_path=res_dir, upload_id=mp_upload_id, uploaded_parts=parts, ) list_parts_res = bucket_utils.list_uploaded_parts( s3_obj=mcg_obj, bucketname=ns_bucket, object_key=mpu_key, upload_id=mp_upload_id, ) logger.info(f"Listing individual parts: {list_parts_res['Parts']}") for i, ele in enumerate(uploaded_parts): assert (ele["PartNumber"] == list_parts_res["Parts"][i] ["PartNumber"]), "Invalid part_number" assert ele["ETag"] == list_parts_res["Parts"][i][ "ETag"], "Invalid ETag" logger.info(f"Completing the Multipart Upload on bucket: {ns_bucket}") assert bucket_utils.complete_multipart_upload( s3_obj=mcg_obj, bucketname=ns_bucket, object_key=mpu_key, upload_id=mp_upload_id, parts=uploaded_parts, ), "MPU did not complete" # Checksum validation after completing MPU logger.info( f"Downloading the completed multipart object from {ns_bucket} to aws-cli pod" ) bucket_utils.sync_object_directory(podobj=awscli_pod, src=object_path, target=res_dir, s3_obj=mcg_obj) assert bucket_utils.verify_s3_object_integrity( original_object_path=f"{origin_dir}/{mpu_key}", result_object_path=f"{res_dir}/{mpu_key}", awscli_pod=awscli_pod, ), "Checksum comparision between original and result object failed"