def test_mcg_namespace_basic_s3_ops_crd(self, mcg_obj, cld_mgr,
bucket_factory, bucketclass_dict):
"""
Test basic S3 operations on namespace buckets.
1. Validates put, get, copy, head, get_acl, delete object operations
2. Validates listObjects v1 and v2 with prefix, delimiter combinations with page entries
"""
max_keys = 50
ns_buc = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0]
ns_bucket = ns_buc.name
# Put, Get, Copy, Head, Get Acl and Delete object operations
logger.info(f"Put and Get object operation on {ns_bucket}")
assert bucket_utils.s3_put_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ,
data=OBJ_DATA), "Failed: PutObject"
get_res = bucket_utils.s3_get_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ)
list_response = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_response, [ROOT_OBJ], "Contents")
assert bucket_utils.s3_copy_object(
s3_obj=mcg_obj,
bucketname=ns_bucket,
source=f"/{ns_bucket}/{ROOT_OBJ}",
object_key=COPY_OBJ,
), "Failed: CopyObject"
get_copy_res = bucket_utils.s3_get_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=COPY_OBJ)
logger.info(f"Verifying Etag of {COPY_OBJ} from Get object operations")
assert get_copy_res["ETag"] == get_res["ETag"], "Incorrect object key"
assert bucket_utils.s3_head_object(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ,
if_match=get_res["ETag"],
), "ETag does not match with the head object"
get_acl_res = bucket_utils.s3_get_object_acl(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ)
logger.info(
f"Verifying Get object ACl response: {get_acl_res['Grants']}")
assert (get_acl_res["Grants"][0]["Grantee"]["ID"] ==
get_acl_res["Owner"]["ID"]), "Invalid Grant ID"
logger.info(
f"Deleting {ROOT_OBJ} and {COPY_OBJ} and verifying response")
del_res = bucket_utils.s3_delete_objects(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_keys=[{
"Key": f"{ROOT_OBJ}"
}, {
"Key": f"{COPY_OBJ}"
}],
)
for i, key in enumerate([ROOT_OBJ, COPY_OBJ]):
assert (key == del_res["Deleted"][i]["Key"]
), "Object key not found/not-deleted"
logger.info("Setting up objects to verify list operations")
obj_keys, obj_prefixes, mid_index = setup_objects_to_list(
amount=100,
prefix="Drive/Folder",
bucket_name=ns_bucket,
mcg_obj=mcg_obj,
)
# List v1 and page entries
logger.info(f"ListObjectsV1 operation on {ns_bucket}")
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_v1_res, obj_keys, "Contents", version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1")
first_page_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys)
last_key = get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
version="v1")
next_page_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys,
marker=last_key)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
version="v1")
# List v1 with prefix and page entries
logger.info(f"ListObjectsV1 operation on {ns_bucket} with prefix")
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/")
get_list_and_verify(list_v1_res,
obj_keys,
"Contents",
"Drive/",
version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1 with prefix"
)
first_page_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys)
last_key = get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
"Drive/",
version="v1")
next_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys,
marker=last_key,
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
"Drive/",
version="v1")
# List v1 with prefix, delimiter and page entries
logger.info(
f"ListObjectsV1 operation on {ns_bucket} with prefix and delimiter"
)
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/")
get_list_and_verify(list_v1_res,
obj_prefixes,
"CommonPrefixes",
"Drive/",
"/",
version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1 with prefix and delimiter"
)
first_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
)
get_list_and_verify(
first_page_res,
obj_prefixes[:mid_index],
"CommonPrefixes",
"Drive/",
"/",
version="v1",
)
next_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
marker=first_page_res["NextMarker"],
)
get_list_and_verify(
next_page_res,
obj_prefixes[mid_index:],
"CommonPrefixes",
"Drive/",
"/",
version="v1",
)
# List v2
logger.info(f"ListObjectsV2 operation on {ns_bucket}")
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_v2_res, obj_keys, "Contents", version="v2")
logger.info(
"Get and verify next page entries of list using ListObjectV2")
first_page_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys)
get_list_and_verify(first_page_res, obj_keys, "Contents", version="v2")
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
version="v2")
# List v2 with prefix
logger.info(f"ListObjectsV2 operation on {ns_bucket} with prefix")
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/")
get_list_and_verify(list_v2_res,
obj_keys,
"Contents",
"Drive/",
version="v2")
logger.info(
"Get and verify next page entries of list using ListObjectV2 with prefix"
)
first_page_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys)
get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
"Drive/",
version="v2")
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
"Drive/",
version="v2")
# List v2 with prefix and delimiter
logger.info(
f"ListObjectsV2 operation on {ns_bucket} with prefix and delimiter"
)
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/")
get_list_and_verify(list_v2_res,
obj_prefixes,
"CommonPrefixes",
"Drive/",
"/",
version="v2")
logger.info(
"Get and verify next page entries of ListObjectV2 with prefix and delimiter"
)
first_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
)
get_list_and_verify(
first_page_res,
obj_prefixes[:mid_index],
"CommonPrefixes",
"Drive/",
"/",
version="v2",
)
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(
next_page_res,
obj_prefixes[mid_index:],
"CommonPrefixes",
"Drive/",
"/",
version="v2",
)
def test_mcg_namespace_basic_s3_ops(self, mcg_obj, ns_resource_factory,
bucket_factory, platform):
"""
Test basic S3 operations on namespace buckets.
1. Validates put, get, copy, head, get_acl, delete object operations
2. Validates listObjects v1 and v2 with prefix, delimiter combinations with page entries
"""
max_keys = 50
namespace_res = ns_resource_factory(platform=platform)
ns_bucket = bucket_factory(
amount=1,
interface="mcg-namespace",
write_ns_resource=namespace_res[1],
read_ns_resources=[namespace_res[1]],
)[0].name
# Put, Get, Copy, Head, Get Acl and Delete object operations
logger.info(f"Put and Get object operation on {ns_bucket}")
assert bucket_utils.s3_put_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ,
data=OBJ_DATA), "Failed: PutObject"
get_res = bucket_utils.s3_get_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ)
list_response = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_response, [ROOT_OBJ], "Contents")
assert bucket_utils.s3_copy_object(
s3_obj=mcg_obj,
bucketname=ns_bucket,
source=f"/{ns_bucket}/{ROOT_OBJ}",
object_key=COPY_OBJ,
), "Failed: CopyObject"
get_copy_res = bucket_utils.s3_get_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=COPY_OBJ)
logger.info(f"Verifying Etag of {COPY_OBJ} from Get object operations")
assert get_copy_res["ETag"] == get_res["ETag"], "Incorrect object key"
head_res = bucket_utils.s3_head_object(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ)
logger.info(
f"Verifying metadata from head_object operation: {head_res['Metadata']}"
)
if platform == constants.AZURE_PLATFORM:
assert (head_res["Metadata"]["noobaa-namespace-blob-container"] ==
namespace_res[0]), "Invalid object metadata"
else:
assert (head_res["Metadata"]["noobaa-namespace-s3-bucket"] ==
namespace_res[0]), "Invalid object metadata"
get_acl_res = bucket_utils.s3_get_object_acl(s3_obj=mcg_obj,
bucketname=ns_bucket,
object_key=ROOT_OBJ)
logger.info(
f"Verifying Get object ACl response: {get_acl_res['Grants']}")
assert (get_acl_res["Grants"][0]["Grantee"]["DisplayName"] == "NooBaa"
), "Invalid grantee"
logger.info(
f"Deleting {ROOT_OBJ} and {COPY_OBJ} and verifying response")
del_res = bucket_utils.s3_delete_objects(
s3_obj=mcg_obj,
bucketname=ns_bucket,
object_keys=[{
"Key": f"{ROOT_OBJ}"
}, {
"Key": f"{COPY_OBJ}"
}],
)
for i, key in enumerate([ROOT_OBJ, COPY_OBJ]):
assert (key == del_res["Deleted"][i]["Key"]
), "Object key not found/not-deleted"
if not platform == constants.AZURE_PLATFORM:
logger.info("Setting up objects to verify list operations")
obj_keys, obj_prefixes, mid_index = setup_objects_to_list(
amount=100,
prefix="Drive/Folder",
bucket_name=ns_bucket,
mcg_obj=mcg_obj,
)
# List v1 and page entries
logger.info(f"ListObjectsV1 operation on {ns_bucket}")
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_v1_res,
obj_keys,
"Contents",
version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1")
first_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj, bucketname=ns_bucket, max_keys=max_keys)
last_key = get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
version="v1")
next_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys,
marker=last_key)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
version="v1")
# List v1 with prefix and page entries
logger.info(f"ListObjectsV1 operation on {ns_bucket} with prefix")
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/")
get_list_and_verify(list_v1_res,
obj_keys,
"Contents",
"Drive/",
version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1 with prefix"
)
first_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys)
last_key = get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
"Drive/",
version="v1")
next_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys,
marker=last_key,
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
"Drive/",
version="v1")
# List v1 with prefix, delimiter and page entries
logger.info(
f"ListObjectsV1 operation on {ns_bucket} with prefix and delimiter"
)
list_v1_res = bucket_utils.s3_list_objects_v1(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/")
get_list_and_verify(list_v1_res,
obj_prefixes,
"CommonPrefixes",
"Drive/",
"/",
version="v1")
logger.info(
"Get and verify next page entries of list using ListObjectV1 with prefix and delimiter"
)
first_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
)
get_list_and_verify(
first_page_res,
obj_prefixes[:mid_index],
"CommonPrefixes",
"Drive/",
"/",
version="v1",
)
next_page_res = bucket_utils.s3_list_objects_v1(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
marker=first_page_res["NextMarker"],
)
get_list_and_verify(
next_page_res,
obj_prefixes[mid_index:],
"CommonPrefixes",
"Drive/",
"/",
version="v1",
)
# List v2
logger.info(f"ListObjectsV2 operation on {ns_bucket}")
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket)
get_list_and_verify(list_v2_res,
obj_keys,
"Contents",
version="v2")
logger.info(
"Get and verify next page entries of list using ListObjectV2")
first_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj, bucketname=ns_bucket, max_keys=max_keys)
get_list_and_verify(first_page_res,
obj_keys,
"Contents",
version="v2")
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
version="v2")
# List v2 with prefix
logger.info(f"ListObjectsV2 operation on {ns_bucket} with prefix")
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/")
get_list_and_verify(list_v2_res,
obj_keys,
"Contents",
"Drive/",
version="v2")
logger.info(
"Get and verify next page entries of list using ListObjectV2 with prefix"
)
first_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys)
get_list_and_verify(first_page_res,
obj_keys[:mid_index],
"Contents",
"Drive/",
version="v2")
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(next_page_res,
obj_keys[mid_index:],
"Contents",
"Drive/",
version="v2")
# List v2 with prefix and delimiter
logger.info(
f"ListObjectsV2 operation on {ns_bucket} with prefix and delimiter"
)
list_v2_res = bucket_utils.s3_list_objects_v2(s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/")
get_list_and_verify(list_v2_res,
obj_prefixes,
"CommonPrefixes",
"Drive/",
"/",
version="v2")
logger.info(
"Get and verify next page entries of list using ListObjectV2 with prefix and delimiter"
)
first_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
)
get_list_and_verify(
first_page_res,
obj_prefixes[:mid_index],
"CommonPrefixes",
"Drive/",
"/",
version="v2",
)
next_page_res = bucket_utils.s3_list_objects_v2(
s3_obj=mcg_obj,
bucketname=ns_bucket,
prefix="Drive/",
delimiter="/",
max_keys=max_keys,
con_token=first_page_res["NextContinuationToken"],
)
get_list_and_verify(
next_page_res,
obj_prefixes[mid_index:],
"CommonPrefixes",
"Drive/",
"/",
version="v2",
)