def test_mcg_namespace_disruptions_crd(
self,
mcg_obj,
cld_mgr,
awscli_pod,
bucketclass_dict,
bucket_factory,
node_drain_teardown,
):
"""
Test MCG namespace disruption flow
1. Create NS resources with CRDs
2. Create NS bucket with CRDs
3. Upload to NS bucket
4. Delete noobaa related pods and verify integrity of objects
5. Create public access policy on NS bucket and verify Get op
6. Drain nodes containing noobaa pods and verify integrity of objects
7. Perform put operation to validate public access denial
7. Edit/verify and remove objects on NS bucket
"""
data = "Sample string content to write to a S3 object"
object_key = "ObjKey-" + str(uuid.uuid4().hex)
awscli_node_name = awscli_pod.get()["spec"]["nodeName"]
aws_s3_creds = {
"access_key_id": cld_mgr.aws_client.access_key,
"access_key": cld_mgr.aws_client.secret_key,
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"region": config.ENV_DATA["region"],
}
# S3 account details
user_name = "nb-user" + str(uuid.uuid4().hex)
email = user_name + "@mail.com"
logger.info("Setting up test files for upload, to the bucket/resources")
setup_base_objects(awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3)
# Create the namespace resource and verify health
ns_buc = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0]
ns_bucket = ns_buc.name
aws_target_bucket = ns_buc.bucketclass.namespacestores[0].uls_name
logger.info(f"Namespace bucket: {ns_bucket} created")
logger.info(f"Uploading objects to ns bucket: {ns_bucket}")
sync_object_directory(
awscli_pod,
src=MCG_NS_ORIGINAL_DIR,
target=f"s3://{ns_bucket}",
s3_obj=mcg_obj,
)
for pod_to_respin in self.labels_map:
logger.info(f"Re-spinning mcg resource: {self.labels_map[pod_to_respin]}")
pod_obj = pod.Pod(
**pod.get_pods_having_label(
label=self.labels_map[pod_to_respin],
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
)[0]
)
pod_obj.delete(force=True)
assert pod_obj.ocp.wait_for_resource(
condition=constants.STATUS_RUNNING,
selector=self.labels_map[pod_to_respin],
resource_count=1,
timeout=300,
)
logger.info(
f"Downloading objects from ns bucket: {ns_bucket} "
f"after re-spinning: {self.labels_map[pod_to_respin]}"
)
sync_object_directory(
awscli_pod,
src=f"s3://{ns_bucket}",
target=MCG_NS_RESULT_DIR,
s3_obj=mcg_obj,
)
logger.info(
f"Verifying integrity of objects "
f"after re-spinning: {self.labels_map[pod_to_respin]}"
)
compare_directory(
awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3
)
# S3 account
user = NoobaaAccount(mcg_obj, name=user_name, email=email, buckets=[ns_bucket])
logger.info(f"Noobaa account: {user.email_id} with S3 access created")
# Admin sets Public access policy(*)
bucket_policy_generated = gen_bucket_policy(
user_list=["*"],
actions_list=["GetObject"],
resources_list=[f'{ns_bucket}/{"*"}'],
)
bucket_policy = json.dumps(bucket_policy_generated)
logger.info(
f"Creating bucket policy on bucket: {ns_bucket} with wildcard (*) Principal"
)
put_policy = put_bucket_policy(mcg_obj, ns_bucket, bucket_policy)
logger.info(f"Put bucket policy response from Admin: {put_policy}")
logger.info(f"Getting bucket policy on bucket: {ns_bucket}")
get_policy = get_bucket_policy(mcg_obj, ns_bucket)
logger.info(f"Got bucket policy: {get_policy['Policy']}")
# MCG admin writes an object to bucket
logger.info(f"Writing object on bucket: {ns_bucket} by admin")
assert s3_put_object(mcg_obj, ns_bucket, object_key, data), "Failed: PutObject"
# Verifying whether Get operation is allowed to any S3 user
logger.info(
f"Get object action on namespace bucket: {ns_bucket} "
f"with user: {user.email_id}"
)
assert s3_get_object(user, ns_bucket, object_key), "Failed: GetObject"
# Upload files to NS target
logger.info(
f"Uploading objects directly to ns resource target: {aws_target_bucket}"
)
sync_object_directory(
awscli_pod,
src=MCG_NS_ORIGINAL_DIR,
target=f"s3://{aws_target_bucket}",
signed_request_creds=aws_s3_creds,
)
for pod_to_drain in self.labels_map:
pod_obj = pod.Pod(
**pod.get_pods_having_label(
label=self.labels_map[pod_to_drain],
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
)[0]
)
# Retrieve the node name on which the pod resides
node_name = pod_obj.get()["spec"]["nodeName"]
if awscli_node_name == node_name:
logger.info(
f"Skipping node drain since aws cli pod node: "
f"{awscli_node_name} is same as {pod_to_drain} "
f"pod node: {node_name}"
)
continue
# Drain the node
drain_nodes([node_name])
wait_for_nodes_status(
[node_name], status=constants.NODE_READY_SCHEDULING_DISABLED
)
schedule_nodes([node_name])
wait_for_nodes_status(timeout=300)
# Retrieve the new pod
pod_obj = pod.Pod(
**pod.get_pods_having_label(
label=self.labels_map[pod_to_drain],
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
)[0]
)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout=120)
# Verify all storage pods are running
wait_for_storage_pods()
logger.info(
f"Downloading objects from ns bucket: {ns_bucket} "
f"after draining node: {node_name} with pod {pod_to_drain}"
)
sync_object_directory(
awscli_pod,
src=f"s3://{ns_bucket}",
target=MCG_NS_RESULT_DIR,
s3_obj=mcg_obj,
)
logger.info(
f"Verifying integrity of objects "
f"after draining node with pod: {pod_to_drain}"
)
compare_directory(
awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3
)
logger.info(f"Editing the namespace resource bucket: {ns_bucket}")
namespace_bucket_update(
mcg_obj,
bucket_name=ns_bucket,
read_resource=[aws_target_bucket],
write_resource=aws_target_bucket,
)
logger.info(f"Verifying object download after edit on ns bucket: {ns_bucket}")
sync_object_directory(
awscli_pod,
src=f"s3://{ns_bucket}",
target=MCG_NS_RESULT_DIR,
s3_obj=mcg_obj,
)
# Verifying whether Put object action is denied
logger.info(
f"Verifying whether user: {user.email_id} has only public read access"
)
logger.info(f"Removing objects from ns bucket: {ns_bucket}")
rm_object_recursive(awscli_pod, target=ns_bucket, mcg_obj=mcg_obj)
def mcg_system_setup(bucket_amount=5, object_amount=10):
# Create standard MCG buckets
test_buckets = bucket_factory(
amount=bucket_amount,
interface="CLI",
)
uploaded_objects_dir = test_directory_setup.origin_dir
downloaded_obejcts_dir = test_directory_setup.result_dir
test_buckets_pattern = "RandomObject-"
first_bidirectional_pattern = "FirstBidi-"
second_bidirectional_pattern = "SecondBidi-"
cache_pattern = "Cache-"
# Perform a round-trip object verification -
# 1. Generate random objects in uploaded_objects_dir
# 2. Upload the objects to the bucket
# 3. Download the objects from the bucket
# 4. Compare the object checksums in downloaded_obejcts_dir
# with the ones in uploaded_objects_dir
for count, bucket in enumerate(test_buckets):
assert random_object_round_trip_verification(
io_pod=awscli_pod_session,
bucket_name=bucket.name,
upload_dir=uploaded_objects_dir + f"Bucket{count}",
download_dir=downloaded_obejcts_dir + f"Bucket{count}",
amount=object_amount,
pattern=test_buckets_pattern,
mcg_obj=mcg_obj_session,
), "Some or all written objects were not found in the list of downloaded objects"
# E2E TODO: Create RGW kafka notification & see the objects are notified to kafka
# Create two MCG buckets with a bidirectional replication policy
bucketclass = {
"interface": "OC",
"backingstore_dict": {
"aws": [(1, "eu-central-1")]
},
}
first_bidi_bucket_name = bucket_factory(
bucketclass=bucketclass)[0].name
replication_policy = ("basic-replication-rule", first_bidi_bucket_name,
None)
second_bidi_bucket_name = bucket_factory(
1, bucketclass=bucketclass,
replication_policy=replication_policy)[0].name
patch_replication_policy_to_bucket(first_bidi_bucket_name,
"basic-replication-rule-2",
second_bidi_bucket_name)
bidi_uploaded_objs_dir_1 = uploaded_objects_dir + "/bidi_1"
bidi_uploaded_objs_dir_2 = uploaded_objects_dir + "/bidi_2"
bidi_downloaded_objs_dir_1 = downloaded_obejcts_dir + "/bidi_1"
bidi_downloaded_objs_dir_2 = downloaded_obejcts_dir + "/bidi_2"
# Verify replication is working as expected by performing a two-way round-trip object verification
random_object_round_trip_verification(
io_pod=awscli_pod_session,
bucket_name=first_bidi_bucket_name,
upload_dir=bidi_uploaded_objs_dir_1,
download_dir=bidi_downloaded_objs_dir_1,
amount=object_amount,
pattern=first_bidirectional_pattern,
wait_for_replication=True,
second_bucket_name=second_bidi_bucket_name,
mcg_obj=mcg_obj_session,
)
random_object_round_trip_verification(
io_pod=awscli_pod_session,
bucket_name=second_bidi_bucket_name,
upload_dir=bidi_uploaded_objs_dir_2,
download_dir=bidi_downloaded_objs_dir_2,
amount=object_amount,
pattern=second_bidirectional_pattern,
wait_for_replication=True,
second_bucket_name=first_bidi_bucket_name,
mcg_obj=mcg_obj_session,
)
# Create a cache bucket
cache_bucketclass = {
"interface": "OC",
"namespace_policy_dict": {
"type": "Cache",
"ttl": 3600000,
"namespacestore_dict": {
"aws": [(1, "eu-central-1")],
},
},
"placement_policy": {
"tiers": [{
"backingStores": [constants.DEFAULT_NOOBAA_BACKINGSTORE]
}]
},
}
cache_bucket = bucket_factory(bucketclass=cache_bucketclass)[0]
cache_uploaded_objs_dir = uploaded_objects_dir + "/cache"
cache_uploaded_objs_dir_2 = uploaded_objects_dir + "/cache_2"
cache_downloaded_objs_dir = downloaded_obejcts_dir + "/cache"
underlying_bucket_name = cache_bucket.bucketclass.namespacestores[
0].uls_name
# Upload a random object to the bucket
objs_written_to_cache_bucket = write_random_test_objects_to_bucket(
awscli_pod_session,
cache_bucket.name,
cache_uploaded_objs_dir,
pattern=cache_pattern,
mcg_obj=mcg_obj_session,
)
wait_for_cache(mcg_obj_session, cache_bucket.name,
objs_written_to_cache_bucket)
# Write a random, larger object directly to the underlying storage of the bucket
write_random_test_objects_to_bucket(
awscli_pod_session,
underlying_bucket_name,
cache_uploaded_objs_dir_2,
pattern=cache_pattern,
s3_creds=cld_mgr.aws_client.nss_creds,
)
# Download the object from the cache bucket
sync_object_directory(
awscli_pod_session,
f"s3://{cache_bucket.name}",
cache_downloaded_objs_dir,
mcg_obj_session,
)
# Make sure the cached object was returned, and not the one that was written to the underlying storage
assert compare_directory(
awscli_pod_session,
cache_uploaded_objs_dir,
cache_downloaded_objs_dir,
amount=1,
pattern=cache_pattern,
), "The uploaded and downloaded cached objects have different checksums"
assert (
compare_directory(
awscli_pod_session,
cache_uploaded_objs_dir_2,
cache_downloaded_objs_dir,
amount=1,
pattern=cache_pattern,
) is False
), "The cached object was replaced by the new one before the TTL has expired"
return {
"test_buckets": test_buckets,
"test_buckets_upload_dir": uploaded_objects_dir,
"object_amount": object_amount,
"test_buckets_pattern": test_buckets_pattern,
"first_bidi_bucket_name": first_bidi_bucket_name,
"bidi_downloaded_objs_dir_2": bidi_downloaded_objs_dir_2,
"first_bidirectional_pattern": first_bidirectional_pattern,
"second_bidi_bucket_name": second_bidi_bucket_name,
"second_bidirectional_pattern": second_bidirectional_pattern,
"cache_bucket_name": cache_bucket.name,
"cache_pattern": cache_pattern,
"cache_downloaded_objs_dir": cache_downloaded_objs_dir,
}
