def test_multipart_with_policy(self, mcg_obj, bucket_factory):
"""
Test Multipart upload with bucket policy set on the bucket
"""
bucket = bucket_factory(interface="OC")[0].name
obc_obj = OBC(bucket)
part_body = "Random data-" + str(uuid.uuid4().hex)
object_key = "MpuObjKey"
bucket_policy_generated = gen_bucket_policy(
user_list=obc_obj.obc_account,
actions_list=[
"ListBucketMultipartUploads",
"ListMultipartUploadParts",
"PutObject",
],
resources_list=[
obc_obj.bucket_name, f'{obc_obj.bucket_name}/{"*"}'
],
effect="Allow",
)
bucket_policy = json.dumps(bucket_policy_generated)
# Creates and gets policy
logger.info(f"Creating policy on bucket: {obc_obj.bucket_name}")
put_bucket_policy(mcg_obj, obc_obj.bucket_name, bucket_policy)
logger.info(f"Getting Bucket policy on bucket: {obc_obj.bucket_name}")
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
logger.info(
f"Initiating MP Upload on Bucket: {bucket} with Key {object_key}")
upload_id = create_multipart_upload(obc_obj, bucket, object_key)
logger.info(
f"Listing the MP Upload : {list_multipart_upload(obc_obj, bucket)['Uploads']}"
)
# Uploading individual part with no body to the Bucket
logger.info(f"Uploading to the bucket: {bucket}")
part_etag = obc_obj.s3_client.upload_part(
Bucket=bucket,
Key=object_key,
Body=part_body,
UploadId=upload_id,
PartNumber=1,
)["ETag"]
# Listing the Uploaded part
logger.info(
f"Listing the individual part: {list_uploaded_parts(obc_obj, bucket, object_key, upload_id)['Parts']}"
)
uploaded_part = [{"ETag": part_etag, "PartNumber": 1}]
# Completing the Multipart Upload
logger.info(f"Completing the MP Upload with on bucket: {bucket}")
complete_multipart_upload(obc_obj, bucket, object_key, upload_id,
uploaded_part)
def test_multipart_upload_operations(self, rgw_endpoint,
awscli_pod_session,
rgw_bucket_factory,
test_directory_setup):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod_session, rgw_bucket_factory, test_directory_setup)
bucketname = bucket.name
bucket = OBC(bucketname)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f"Aborting any Multipart Upload on bucket:{bucketname}")
abort_all_multipart_upload(bucket, bucketname, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f"Initiating Multipart Upload on Bucket: {bucketname} with Key {key}"
)
upload_id = create_multipart_upload(bucket, bucketname, key)
logger.info(
f"Listing the Multipart Upload: {list_multipart_upload(bucket, bucketname)}"
)
# Uploading individual parts to the Bucket
logger.info(f"Uploading individual parts to the bucket {bucketname}")
uploaded_parts = upload_parts(bucket, awscli_pod_session, bucketname,
key, res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f"Listing the individual parts: {list_uploaded_parts(bucket, bucketname, key, upload_id)}"
)
# Completing the Multipart Upload
logger.info(f"Completing the Multipart Upload on bucket: {bucketname}")
logger.info(
complete_multipart_upload(bucket, bucketname, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
"Downloading the completed multipart object from the RGW bucket to the awscli pod"
)
sync_object_directory(awscli_pod_session, object_path, res_dir, bucket)
assert verify_s3_object_integrity(
original_object_path=f"{origin_dir}/{key}",
result_object_path=f"{res_dir}/{key}",
awscli_pod=awscli_pod_session,
), "Checksum comparision between original and result object failed"
def test_multipart_upload_operations(self, mcg_obj, awscli_pod,
bucket_factory):
"""
Test Multipart upload operations on bucket and verifies the integrity of the downloaded object
"""
bucket, key, origin_dir, res_dir, object_path, parts = setup(
awscli_pod, bucket_factory)
# Abort all Multipart Uploads for this Bucket (optional, for starting over)
logger.info(f'Aborting any Multipart Upload on bucket:{bucket}')
abort_all_multipart_upload(mcg_obj, bucket, key)
# Create & list Multipart Upload on the Bucket
logger.info(
f'Initiating Multipart Upload on Bucket: {bucket} with Key {key}')
upload_id = create_multipart_upload(mcg_obj, bucket, key)
logger.info(
f'Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)}'
)
# Uploading individual parts to the Bucket
logger.info(f'Uploading individual parts to the bucket {bucket}')
uploaded_parts = upload_parts(mcg_obj, awscli_pod, bucket, key,
res_dir, upload_id, parts)
# Listing the Uploaded parts
logger.info(
f'Listing the individual parts : {list_uploaded_parts(mcg_obj, bucket, key, upload_id)}'
)
# Completing the Multipart Upload
logger.info(f'Completing the Multipart Upload on bucket: {bucket}')
logger.info(
complete_multipart_upload(mcg_obj, bucket, key, upload_id,
uploaded_parts))
# Checksum Validation: Downloading the object after completing Multipart Upload and verifying its integrity
logger.info(
'Downloading the completed multipart object from MCG bucket to awscli pod'
)
sync_object_directory(awscli_pod, object_path, res_dir, mcg_obj)
assert verify_s3_object_integrity(
original_object_path=f'{origin_dir}/{key}',
result_object_path=f'{res_dir}/{key}',
awscli_pod=awscli_pod
), 'Checksum comparision between original and result object failed'
def test_multipart_with_no_body(self, mcg_obj, bucket_factory):
"""
Test Multipart upload with no body while uploading the part
"""
bucket = bucket_factory(amount=1, interface="OC")[0].name
object_key = "ObjKey-" + str(uuid.uuid4().hex)
# Create & list Multipart Upload on the Bucket
logger.info(
f"Initiating Multipart Upload on Bucket: {bucket} with Key {object_key}"
)
upload_id = create_multipart_upload(mcg_obj, bucket, object_key)
logger.info(
f"Listing the Multipart Upload : {list_multipart_upload(mcg_obj, bucket)['Uploads']}"
)
# Uploading individual part with no body to the Bucket
logger.info(
f"Uploading individual parts to the bucket: {bucket} with no body specified"
)
part_etag = mcg_obj.s3_client.upload_part(Bucket=bucket,
Key=object_key,
UploadId=upload_id,
PartNumber=1)["ETag"]
uploaded_part = [{"ETag": part_etag, "PartNumber": 1}]
# Listing the Uploaded part
logger.info(
f"Listing the individual part: {list_uploaded_parts(mcg_obj, bucket, object_key, upload_id)['Parts']}"
)
# Completing the Multipart Upload
logger.info(
f"Completing the Multipart Upload with a part and no body on bucket: {bucket}"
)
complete_multipart_upload(mcg_obj, bucket, object_key, upload_id,
uploaded_part)
def test_mcg_namespace_mpu_crd(self, mcg_obj, awscli_pod, bucket_factory,
bucketclass_dict):
"""
Test multipart upload S3 operations on namespace buckets(created by CRDs)
Validates create, upload, upload copy and list parts operations
"""
ns_buc = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0]
ns_bucket = ns_buc.name
object_path = f"s3://{ns_bucket}"
logger.info(
f"Setting up test files for mpu and aborting any mpu on bucket: {ns_bucket}"
)
mpu_key, origin_dir, res_dir, parts = multipart_setup(awscli_pod)
bucket_utils.abort_all_multipart_upload(mcg_obj, ns_bucket, COPY_OBJ)
# Initiate mpu, Upload part copy, List and Abort operations
logger.info(
f"Put object on bucket: {ns_bucket} to create a copy source")
assert bucket_utils.s3_put_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ,
data=OBJ_DATA), "Failed: PutObject"
logger.info(
f"Initiating mpu on bucket: {ns_bucket} with key {COPY_OBJ}")
part_copy_id = bucket_utils.create_multipart_upload(
mcg_obj, ns_bucket, COPY_OBJ)
list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj,
bucketname=ns_bucket)
if (constants.AZURE_PLATFORM
not in bucketclass_dict["namespace_policy_dict"]
["namespacestore_dict"]):
logger.info(f"Listing in-progress mpu: {list_mpu_res}")
assert (part_copy_id == list_mpu_res["Uploads"][0]["UploadId"]
), "Invalid UploadId"
logger.info(f"Uploading a part copy to: {ns_bucket}")
assert bucket_utils.s3_upload_part_copy(
s3_obj=mcg_obj,
bucketname=ns_bucket,
copy_source=f"/{ns_bucket}/{ROOT_OBJ}",
object_key=COPY_OBJ,
part_number=1,
upload_id=part_copy_id,
), "Failed: upload part copy"
logger.info(
f"Aborting initiated multipart upload with id: {part_copy_id}")
assert bucket_utils.abort_multipart(mcg_obj, ns_bucket, COPY_OBJ,
part_copy_id), "Abort failed"
# Initiate mpu, Upload part, List parts operations
logger.info(
f"Initiating Multipart Upload on Bucket: {ns_bucket} with Key: {mpu_key}"
)
mp_upload_id = bucket_utils.create_multipart_upload(
mcg_obj, ns_bucket, mpu_key)
list_mpu_res = bucket_utils.list_multipart_upload(s3_obj=mcg_obj,
bucketname=ns_bucket)
if (constants.AZURE_PLATFORM
not in bucketclass_dict["namespace_policy_dict"]
["namespacestore_dict"]):
logger.info(f"Listing multipart upload: {list_mpu_res}")
assert (mp_upload_id == list_mpu_res["Uploads"][0]["UploadId"]
), "Invalid UploadId"
logger.info(f"Uploading individual parts to the bucket: {ns_bucket}")
uploaded_parts = bucket_utils.upload_parts(
mcg_obj=mcg_obj,
awscli_pod=awscli_pod,
bucketname=ns_bucket,
object_key=mpu_key,
body_path=res_dir,
upload_id=mp_upload_id,
uploaded_parts=parts,
)
list_parts_res = bucket_utils.list_uploaded_parts(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=mpu_key,
upload_id=mp_upload_id,
)
logger.info(f"Listing individual parts: {list_parts_res['Parts']}")
for i, ele in enumerate(uploaded_parts):
assert (ele["PartNumber"] == list_parts_res["Parts"][i]
["PartNumber"]), "Invalid part_number"
assert ele["ETag"] == list_parts_res["Parts"][i][
"ETag"], "Invalid ETag"
logger.info(f"Completing the Multipart Upload on bucket: {ns_bucket}")
assert bucket_utils.complete_multipart_upload(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=mpu_key,
upload_id=mp_upload_id,
parts=uploaded_parts,
), "MPU did not complete"
# Checksum validation after completing MPU
logger.info(
f"Downloading the completed multipart object from {ns_bucket} to aws-cli pod"
)
bucket_utils.sync_object_directory(podobj=awscli_pod,
src=object_path,
target=res_dir,
s3_obj=mcg_obj)
assert bucket_utils.verify_s3_object_integrity(
original_object_path=f"{origin_dir}/{mpu_key}",
result_object_path=f"{res_dir}/{mpu_key}",
awscli_pod=awscli_pod,
), "Checksum comparision between original and result object failed"
def test_object_actions(self, mcg_obj, bucket_factory):
"""
Test to verify different object actions and cross account access to buckets
"""
data = "Sample string content to write to a new S3 object"
object_key = "ObjKey-" + str(uuid.uuid4().hex)
# Creating multiple obc users (accounts)
obc = bucket_factory(amount=1, interface='OC')
obc_obj = OBC(obc[0].name)
# Admin sets policy on obc bucket with obc account principal
bucket_policy_generated = gen_bucket_policy(
user_list=obc_obj.obc_account,
actions_list=['PutObject'],
resources_list=[f'{obc_obj.bucket_name}/{"*"}'])
bucket_policy = json.dumps(bucket_policy_generated)
logger.info(
f'Creating bucket policy on bucket: {obc_obj.bucket_name} with principal: {obc_obj.obc_account}'
)
put_policy = put_bucket_policy(mcg_obj, obc_obj.bucket_name,
bucket_policy)
logger.info(f'Put bucket policy response from Admin: {put_policy}')
# Get Policy
logger.info(f'Getting Bucket policy on bucket: {obc_obj.bucket_name}')
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
# Verifying whether obc account can put object
logger.info(f'Adding object on bucket: {obc_obj.bucket_name}')
assert s3_put_object(obc_obj, obc_obj.bucket_name, object_key,
data), "Failed: Put Object"
# Verifying whether Get action is not allowed
logger.info(
f'Verifying whether user: {obc_obj.obc_account} is denied to Get object'
)
try:
s3_get_object(obc_obj, obc_obj.bucket_name, object_key)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error['Code'] == 'AccessDenied':
logger.info('Get Object action has been denied access')
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
# Verifying whether obc account allowed to create multipart
logger.info(
f'Creating multipart on bucket: {obc_obj.bucket_name} with key: {object_key}'
)
create_multipart_upload(obc_obj, obc_obj.bucket_name, object_key)
# Verifying whether obc account is denied access to delete object
logger.info(
f'Verifying whether user: {obc_obj.obc_account} is denied to Delete object'
)
try:
s3_delete_object(obc_obj, obc_obj.bucket_name, object_key)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error['Code'] == 'AccessDenied':
logger.info('Delete action has been denied access')
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
# Creating noobaa account to access bucket belonging to obc account
user_name = "noobaa-user" + str(uuid.uuid4().hex)
email = user_name + "@mail.com"
user = NoobaaAccount(mcg_obj,
name=user_name,
email=email,
buckets=[obc_obj.bucket_name])
# Admin sets a policy on obc-account bucket with noobaa-account principal (cross account access)
new_policy_generated = gen_bucket_policy(
user_list=user.email_id,
actions_list=['GetObject', 'DeleteObject'],
resources_list=[f'{obc_obj.bucket_name}/{"*"}'])
new_policy = json.dumps(new_policy_generated)
logger.info(
f'Creating bucket policy on bucket: {obc_obj.bucket_name} with principal: {obc_obj.obc_account}'
)
put_policy = put_bucket_policy(mcg_obj, obc_obj.bucket_name,
new_policy)
logger.info(f'Put bucket policy response from admin: {put_policy}')
# Get Policy
logger.info(f'Getting bucket policy on bucket: {obc_obj.bucket_name}')
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
# Verifying whether Get, Delete object is allowed
logger.info(
f'Getting object on bucket: {obc_obj.bucket_name} with user: {user.email_id}'
)
assert s3_get_object(user, obc_obj.bucket_name,
object_key), "Failed: Get Object"
logger.info(
f'Deleting object on bucket: {obc_obj.bucket_name} with user: {user.email_id}'
)
assert s3_delete_object(user, obc_obj.bucket_name,
object_key), "Failed: Delete Object"
# Verifying whether Put object action is denied
logger.info(
f'Verifying whether user: {user.email_id} is denied to Put object after updating policy'
)
try:
s3_put_object(user, obc_obj.bucket_name, object_key, data)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error['Code'] == 'AccessDenied':
logger.info('Put object action has been denied access')
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
def test_object_actions(self, mcg_obj, bucket_factory):
"""
Test to verify different object actions and cross account access to buckets
"""
data = "Sample string content to write to a new S3 object"
object_key = "ObjKey-" + str(uuid.uuid4().hex)
# Creating multiple obc users (accounts)
obc = bucket_factory(amount=1, interface="OC")
obc_obj = OBC(obc[0].name)
# Creating noobaa account to access bucket belonging to obc account
user_name = "noobaa-user" + str(uuid.uuid4().hex)
email = user_name + "@mail.com"
user = NoobaaAccount(
mcg_obj, name=user_name, email=email, buckets=[obc_obj.bucket_name]
)
# Admin sets policy on obc bucket with obc account principal
bucket_policy_generated = gen_bucket_policy(
user_list=[obc_obj.obc_account, user.email_id],
actions_list=["PutObject"]
if version.get_semantic_ocs_version_from_config() <= version.VERSION_4_6
else ["GetObject", "DeleteObject"],
effect="Allow"
if version.get_semantic_ocs_version_from_config() <= version.VERSION_4_6
else "Deny",
resources_list=[f'{obc_obj.bucket_name}/{"*"}'],
)
bucket_policy = json.dumps(bucket_policy_generated)
logger.info(
f"Creating bucket policy on bucket: {obc_obj.bucket_name} with principal: {obc_obj.obc_account}"
)
put_policy = put_bucket_policy(mcg_obj, obc_obj.bucket_name, bucket_policy)
logger.info(f"Put bucket policy response from Admin: {put_policy}")
# Get Policy
logger.info(f"Getting Bucket policy on bucket: {obc_obj.bucket_name}")
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
# Verifying whether users can put object
logger.info(
f"Adding object on bucket: {obc_obj.bucket_name} using user: {obc_obj.obc_account}"
)
assert s3_put_object(
obc_obj, obc_obj.bucket_name, object_key, data
), "Failed: Put Object"
logger.info(
f"Adding object on bucket: {obc_obj.bucket_name} using user: {user.email_id}"
)
assert s3_put_object(
user, obc_obj.bucket_name, object_key, data
), "Failed: Put Object"
# Verifying whether Get action is not allowed
logger.info(
f"Verifying whether user: "******"ocs_version"]) >= 4.6 else obc_obj.obc_account}'
f" is denied to Get object"
)
try:
if version.get_semantic_ocs_version_from_config() >= version.VERSION_4_6:
s3_get_object(user, obc_obj.bucket_name, object_key)
else:
s3_get_object(obc_obj, obc_obj.bucket_name, object_key)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error["Code"] == "AccessDenied":
logger.info("Get Object action has been denied access")
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
else:
assert False, "Get object succeeded when it should have failed"
if version.get_semantic_ocs_version_from_config() == version.VERSION_4_6:
logger.info(
f"Verifying whether the user: "******"{obc_obj.obc_account} is able to access Get action"
f"irrespective of the policy set"
)
assert s3_get_object(
obc_obj, obc_obj.bucket_name, object_key
), "Failed: Get Object"
# Verifying whether obc account allowed to create multipart
logger.info(
f"Creating multipart on bucket: {obc_obj.bucket_name}"
f" with key: {object_key} using user: {obc_obj.obc_account}"
)
create_multipart_upload(obc_obj, obc_obj.bucket_name, object_key)
# Verifying whether S3 user is allowed to create multipart
logger.info(
f"Creating multipart on bucket: {obc_obj.bucket_name} "
f"with key: {object_key} using user: {user.email_id}"
)
create_multipart_upload(user, obc_obj.bucket_name, object_key)
# Verifying whether obc account is denied access to delete object
logger.info(
f"Verifying whether user: "******"ocs_version"]) >= 4.6 else obc_obj.obc_account}'
f"is denied to Delete object"
)
try:
if version.get_semantic_ocs_version_from_config() >= version.VERSION_4_6:
s3_delete_object(user, obc_obj.bucket_name, object_key)
else:
s3_delete_object(obc_obj, obc_obj.bucket_name, object_key)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error["Code"] == "AccessDenied":
logger.info("Delete action has been denied access")
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
else:
assert False, "Delete object succeeded when it should have failed"
# Admin sets a policy on obc-account bucket with noobaa-account principal (cross account access)
new_policy_generated = gen_bucket_policy(
user_list=[user.email_id],
actions_list=["GetObject", "DeleteObject"]
if float(config.ENV_DATA["ocs_version"]) <= 4.6
else ["PutObject"],
effect="Allow"
if version.get_semantic_ocs_version_from_config() >= version.VERSION_4_6
else "Deny",
resources_list=[f'{obc_obj.bucket_name}/{"*"}'],
)
new_policy = json.dumps(new_policy_generated)
logger.info(
f"Creating bucket policy on bucket: {obc_obj.bucket_name} with principal: {obc_obj.obc_account}"
)
put_policy = put_bucket_policy(mcg_obj, obc_obj.bucket_name, new_policy)
logger.info(f"Put bucket policy response from admin: {put_policy}")
# Get Policy
logger.info(f"Getting bucket policy on bucket: {obc_obj.bucket_name}")
get_policy = get_bucket_policy(mcg_obj, obc_obj.bucket_name)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
# Verifying whether Get, Delete object is allowed
logger.info(
f"Getting object on bucket: {obc_obj.bucket_name} with user: {user.email_id}"
)
for get_resp in TimeoutSampler(
30, 4, s3_get_object, user, obc_obj.bucket_name, object_key
):
if "403" not in str(get_resp["ResponseMetadata"]["HTTPStatusCode"]):
logger.info("GetObj operation successful")
break
else:
logger.info("GetObj operation is denied access")
logger.info(
f"Deleting object on bucket: {obc_obj.bucket_name} with user: {user.email_id}"
)
for del_resp in TimeoutSampler(
30, 4, s3_delete_object, user, obc_obj.bucket_name, object_key
):
if "403" not in str(del_resp["ResponseMetadata"]["HTTPStatusCode"]):
logger.info("DeleteObj operation successful")
break
else:
logger.info("DeleteObj operation is denied access")
# Verifying whether Put object action is denied
logger.info(
f"Verifying whether user: {user.email_id} is denied to Put object after updating policy"
)
try:
s3_put_object(user, obc_obj.bucket_name, object_key, data)
except boto3exception.ClientError as e:
logger.info(e.response)
response = HttpResponseParser(e.response)
if response.error["Code"] == "AccessDenied":
logger.info("Put object action has been denied access")
else:
raise UnexpectedBehaviour(
f"{e.response} received invalid error code {response.error['Code']}"
)
