def test_list_cached_objects( self, bucket_factory, mcg_obj, cld_mgr, awscli_pod_session, test_directory_setup, bucketclass_dict, ): """ Test the ability to list the object stored in a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] original_folder = test_directory_setup.origin_dir # Upload files to NS bucket writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod_session, bucket_to_write=bucket_obj.name, original_dir=original_folder, amount=3, ) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names)
def test_delete_cached_object(self, bucket_factory, mcg_obj, cld_mgr, awscli_pod, bucketclass_dict): """ Test the deletion of an object that is present in the cache of a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] # Upload files to NS bucket writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod, bucket_to_write=bucket_obj.name, amount=1) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names) # Delete the object from mcg interface s3_delete_object(mcg_obj, bucket_obj.name, writen_objs_names[0]) sleep(5) if not check_cached_objects_by_name(mcg_obj, bucket_obj.name): raise UnexpectedBehaviour( "Object was not deleted from cache properly") # Check deletion in the cloud provider aws_target_bucket = bucket_obj.bucketclass.namespacestores[0].uls_name aws_obj_list = list( cld_mgr.aws_client.client.Bucket(aws_target_bucket).objects.all()) if writen_objs_names[0] in aws_obj_list: raise UnexpectedBehaviour( "Object was not deleted from cache properly")
def test_read_non_cached_object(self, bucket_factory, mcg_obj, cld_mgr, awscli_pod, bucketclass_dict): """ Test reading an object that is not present in a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] s3_creds = { "access_key_id": cld_mgr.aws_client.access_key, "access_key": cld_mgr.aws_client.secret_key, "endpoint": constants.MCG_NS_AWS_ENDPOINT, "region": self.DEFAULT_REGION, } aws_target_bucket = bucket_obj.bucketclass.namespacestores[0].uls_name # Upload files directly to AWS writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod, bucket_to_write=aws_target_bucket, amount=3, s3_creds=s3_creds, ) if not check_cached_objects_by_name(mcg_obj, bucket_obj.name): raise UnexpectedBehaviour( "Objects were found in the cache of an empty bucket") # Read files from ns bucket self.download_files(mcg_obj, awscli_pod, bucket_to_read=bucket_obj.name) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names)
def test_read_cached_object( self, bucket_factory, mcg_obj, cld_mgr, awscli_pod_session, test_directory_setup, bucketclass_dict, ): """ Test reading an object that is present in a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] s3_creds = { "access_key_id": cld_mgr.aws_client.access_key, "access_key": cld_mgr.aws_client.secret_key, "endpoint": constants.MCG_NS_AWS_ENDPOINT, "region": self.DEFAULT_REGION, } original_folder = test_directory_setup.origin_dir result_folder = test_directory_setup.result_dir aws_target_bucket = bucket_obj.bucketclass.namespacestores[0].uls_name # Upload files to NS bucket writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod_session, bucket_to_write=bucket_obj.name, original_dir=original_folder, amount=1, ) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names) # Upload files directly to AWS self.write_files_to_pod_and_upload( mcg_obj, awscli_pod_session, bucket_to_write=aws_target_bucket, original_dir=original_folder, amount=1, s3_creds=s3_creds, ) # Read files from ns bucket self.download_files( mcg_obj, awscli_pod_session, bucket_to_read=bucket_obj.name, download_dir=result_folder, ) # Compare dirs should return false since we expect the cached object to return # instead of the new object currently present in the original dir if self.compare_dirs(awscli_pod_session, origin=original_folder, destination=result_folder): raise UnexpectedBehaviour("Cached object was not downloaded")
def test_list_cached_objects(self, bucket_factory, mcg_obj, cld_mgr, awscli_pod, bucketclass_dict): """ Test the ability to list the object stored in a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] # Upload files to NS bucket writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod, bucket_to_write=bucket_obj.name, amount=3) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names)
def test_read_stale_object(self, bucket_factory, mcg_obj, cld_mgr, awscli_pod, bucketclass_dict): """ Test reading a stale object from a cache bucket. """ # Create the cached namespace bucket on top of the namespace resource bucket_obj = bucket_factory(bucketclass=bucketclass_dict)[0] s3_creds = { "access_key_id": cld_mgr.aws_client.access_key, "access_key": cld_mgr.aws_client.secret_key, "endpoint": constants.MCG_NS_AWS_ENDPOINT, "region": self.DEFAULT_REGION, } aws_target_bucket = bucket_obj.bucketclass.namespacestores[0].uls_name # Upload files to NS bucket writen_objs_names = self.write_files_to_pod_and_upload( mcg_obj, awscli_pod, bucket_to_write=bucket_obj.name, amount=1) wait_for_cache(mcg_obj, bucket_obj.name, writen_objs_names) awscli_pod.exec_cmd_on_pod( "mv /original/testfile0.txt /original/testfile1.txt") # Upload files directly to AWS self.write_files_to_pod_and_upload( mcg_obj, awscli_pod, bucket_to_write=aws_target_bucket, amount=1, s3_creds=s3_creds, ) awscli_pod.exec_cmd_on_pod( "mv /original/testfile1.txt /original/testfile0.txt") # using sleep and not TimeoutSampler because we need to wait throughout the whole ttl sleep(bucketclass_dict["namespace_policy_dict"]["ttl"] / 1000) # Read files from ns bucket self.download_files(mcg_obj, awscli_pod, bucket_to_read=bucket_obj.name) if self.compare_dirs(awscli_pod): raise UnexpectedBehaviour( "Updated file was not fetched after ttl was exceeded")
def test_mcg_cache_lifecycle( self, mcg_obj, cld_mgr, awscli_pod, bucket_factory, test_directory_setup, bucketclass_dict, ): """ Test MCG cache bucket lifecycle 1. Create cache buckets on each namespace stores (RGW-OBC/OBC) 2. Verify write operations cache and hub bucket 3. Verify read/list operations on cache bucket and hub target 4. Verify delete operation on buckets 5. Delete multiple cache buckets with data still in ns store 6. Recreate the cache buckets on ns store(with existing data) then read. """ data = "Sample string content to write to a S3 object" object_key = "ObjKey-" + str(uuid.uuid4().hex) if (constants.RGW_PLATFORM in bucketclass_dict["namespace_policy_dict"] ["namespacestore_dict"]): s3_creds = { "access_key_id": cld_mgr.rgw_client.access_key, "access_key": cld_mgr.rgw_client.secret_key, "endpoint": cld_mgr.rgw_client.endpoint, } logger.info("RGW obc will be created as cache bucket") obc_interface = "rgw-oc" else: s3_creds = { "access_key_id": cld_mgr.aws_client.access_key, "access_key": cld_mgr.aws_client.secret_key, "endpoint": constants.MCG_NS_AWS_ENDPOINT, "region": config.ENV_DATA["region"], } logger.info("Noobaa obc will be created as cache bucket") obc_interface = bucketclass_dict["interface"] # Create the namespace resource and bucket ns_bucket = bucket_factory( interface=obc_interface, bucketclass=bucketclass_dict, )[0] logger.info(f"Cache bucket: {ns_bucket.name} created") target_bucket = ns_bucket.bucketclass.namespacestores[0].uls_name # Write to cache logger.info(f"Writing object on cache bucket: {ns_bucket.name}") assert s3_put_object(mcg_obj, ns_bucket.name, object_key, data), "Failed: PutObject" wait_for_cache(mcg_obj, ns_bucket.name, [object_key]) # Write to hub and read from cache logger.info("Setting up test files for upload") setup_base_objects(awscli_pod, test_directory_setup.origin_dir, amount=3) logger.info(f"Uploading objects to ns target: {target_bucket}") sync_object_directory( awscli_pod, src=test_directory_setup.origin_dir, target=f"s3://{target_bucket}", signed_request_creds=s3_creds, ) sync_object_directory( awscli_pod, f"s3://{ns_bucket.name}", test_directory_setup.result_dir, mcg_obj, ) # Read cached object assert s3_get_object(mcg_obj, ns_bucket.name, object_key), "Failed: GetObject" # Read stale object(ttl expired) sleep(bucketclass_dict["namespace_policy_dict"]["ttl"] / 1000) logger.info(f"Get object on cache bucket: {ns_bucket.name}") assert s3_get_object(mcg_obj, ns_bucket.name, object_key), "Failed: GetObject" # List on cache bucket list_response = s3_list_objects_v1(s3_obj=mcg_obj, bucketname=ns_bucket.name) logger.info(f"Listed objects: {list_response}") # Delete object from cache bucket s3_delete_object(mcg_obj, ns_bucket.name, object_key) sleep(5) # Try to read deleted object try: s3_get_object(mcg_obj, ns_bucket.name, object_key) except boto3exception.ClientError: logger.info("object deleted successfully") # Validate deletion on the hub if (constants.RGW_PLATFORM in bucketclass_dict["namespace_policy_dict"] ["namespacestore_dict"]): obj_list = list( cld_mgr.rgw_client.client.Bucket(target_bucket).objects.all()) else: obj_list = list( cld_mgr.aws_client.client.Bucket(target_bucket).objects.all()) if object_key in obj_list: raise UnexpectedBehaviour( "Object was not deleted from cache properly") # Recreate and validate object assert s3_put_object(mcg_obj, ns_bucket.name, object_key, data), "Failed: PutObject" assert s3_get_object(mcg_obj, ns_bucket.name, object_key), "Failed: GetObject" logger.info(f"Deleting cache bucket {ns_bucket.name}") curr_ns_store = ns_bucket.bucketclass.namespacestores[0] ns_bucket.delete() new_bucket_class = { "interface": "OC", "namespace_policy_dict": { "type": "Cache", "ttl": 180000, "namespacestores": [curr_ns_store], }, "placement_policy": { "tiers": [{ "backingStores": [constants.DEFAULT_NOOBAA_BACKINGSTORE] }] }, } logger.info( f"Recreating cache bucket {ns_bucket.name} using current hub: {target_bucket}" ) ns_bucket = bucket_factory( interface=obc_interface, bucketclass=new_bucket_class, )[0] logger.info( f"Read existing data on hub: {target_bucket} through cache bucket: {ns_bucket.name}" ) assert s3_get_object(mcg_obj, ns_bucket.name, object_key), "Failed: GetObject"
def factory(num_of_obcs=20, bulk=False, measure=True): """ Args: num_of_obcs (int) : Number of OBCs we want to create of each type mentioned above. (Total OBCs = num_of_obcs * 5) bulk (bool) : True for bulk operations, False otherwise. measure (bool) : True if we want to measure the OBC creation/deletion time, False otherwise. """ # Create OBCs - bs, ns, cached and create random files obc_objs = list() obc_names = list() obc_params = [ ( "OC", { "interface": "OC", "namespace_policy_dict": { "type": "Single", "namespacestore_dict": { "rgw": [(1, None)] }, }, }, ), ("OC", None), ( "OC", { "interface": "OC", "namespace_policy_dict": { "type": "Cache", "ttl": 3600, "namespacestore_dict": { "rgw": [(1, None)] }, }, "placement_policy": { "tiers": [{ "backingStores": [constants.DEFAULT_NOOBAA_BACKINGSTORE] }] }, }, ), ] for _interface, _bucketclass in obc_params: if num_of_obcs > 0: buckets = bucket_factory( amount=num_of_obcs, interface=_interface, bucketclass=_bucketclass, verify_health=not bulk, ) if bulk: for bucket in buckets: bucket.verify_health() obc_objs.extend(buckets) written_objs_names = write_empty_files_to_bucket( mcg_obj, awscli_pod_session, buckets[0].name, test_directory_setup) if (_bucketclass and _bucketclass["namespace_policy_dict"]["type"] == "Cache"): wait_for_cache(mcg_obj, buckets[0].name, list(written_objs_names)) # Create OBCs - Replica Pair, create random files and verify replication target_bucketclass = { "interface": "OC", "namespace_policy_dict": { "type": "Single", "namespacestore_dict": { "rgw": [(1, None)] }, }, } source_bucketclass = { "interface": "OC", "namespace_policy_dict": { "type": "Single", "namespacestore_dict": { "rgw": [(1, None)] }, }, } target_buckets = list() source_buckets = list() for _num in range(num_of_obcs): target_bucket = bucket_factory(bucketclass=target_bucketclass)[0] target_buckets.append(target_bucket) target_bucket_name = target_bucket.name replication_policy = ("basic-replication-rule", target_bucket_name, None) source_bucket = bucket_factory( 1, bucketclass=source_bucketclass, replication_policy=replication_policy)[0] source_buckets.append(source_bucket) write_empty_files_to_bucket(mcg_obj, awscli_pod_session, source_bucket.name, test_directory_setup) compare_bucket_object_list(mcg_obj_session, source_bucket.name, target_bucket_name) obc_objs.extend(target_buckets) obc_objs.extend(source_buckets) for obc in obc_objs: obc_names.append(obc.name) if measure: # Measure OBC Creation Time scale_noobaa_lib.measure_obc_creation_time(obc_name_list=obc_names) # Delete OBCs for bucket in obc_objs: log.info(f"Deleting bucket: {bucket.name}") bucket.delete() if measure: # Measure OBC Deletion Time scale_noobaa_lib.measure_obc_deletion_time(obc_name_list=obc_names)
def mcg_system_setup(bucket_amount=5, object_amount=10): # Create standard MCG buckets test_buckets = bucket_factory( amount=bucket_amount, interface="CLI", ) uploaded_objects_dir = test_directory_setup.origin_dir downloaded_obejcts_dir = test_directory_setup.result_dir test_buckets_pattern = "RandomObject-" first_bidirectional_pattern = "FirstBidi-" second_bidirectional_pattern = "SecondBidi-" cache_pattern = "Cache-" # Perform a round-trip object verification - # 1. Generate random objects in uploaded_objects_dir # 2. Upload the objects to the bucket # 3. Download the objects from the bucket # 4. Compare the object checksums in downloaded_obejcts_dir # with the ones in uploaded_objects_dir for count, bucket in enumerate(test_buckets): assert random_object_round_trip_verification( io_pod=awscli_pod_session, bucket_name=bucket.name, upload_dir=uploaded_objects_dir + f"Bucket{count}", download_dir=downloaded_obejcts_dir + f"Bucket{count}", amount=object_amount, pattern=test_buckets_pattern, mcg_obj=mcg_obj_session, ), "Some or all written objects were not found in the list of downloaded objects" # E2E TODO: Create RGW kafka notification & see the objects are notified to kafka # Create two MCG buckets with a bidirectional replication policy bucketclass = { "interface": "OC", "backingstore_dict": { "aws": [(1, "eu-central-1")] }, } first_bidi_bucket_name = bucket_factory( bucketclass=bucketclass)[0].name replication_policy = ("basic-replication-rule", first_bidi_bucket_name, None) second_bidi_bucket_name = bucket_factory( 1, bucketclass=bucketclass, replication_policy=replication_policy)[0].name patch_replication_policy_to_bucket(first_bidi_bucket_name, "basic-replication-rule-2", second_bidi_bucket_name) bidi_uploaded_objs_dir_1 = uploaded_objects_dir + "/bidi_1" bidi_uploaded_objs_dir_2 = uploaded_objects_dir + "/bidi_2" bidi_downloaded_objs_dir_1 = downloaded_obejcts_dir + "/bidi_1" bidi_downloaded_objs_dir_2 = downloaded_obejcts_dir + "/bidi_2" # Verify replication is working as expected by performing a two-way round-trip object verification random_object_round_trip_verification( io_pod=awscli_pod_session, bucket_name=first_bidi_bucket_name, upload_dir=bidi_uploaded_objs_dir_1, download_dir=bidi_downloaded_objs_dir_1, amount=object_amount, pattern=first_bidirectional_pattern, wait_for_replication=True, second_bucket_name=second_bidi_bucket_name, mcg_obj=mcg_obj_session, ) random_object_round_trip_verification( io_pod=awscli_pod_session, bucket_name=second_bidi_bucket_name, upload_dir=bidi_uploaded_objs_dir_2, download_dir=bidi_downloaded_objs_dir_2, amount=object_amount, pattern=second_bidirectional_pattern, wait_for_replication=True, second_bucket_name=first_bidi_bucket_name, mcg_obj=mcg_obj_session, ) # Create a cache bucket cache_bucketclass = { "interface": "OC", "namespace_policy_dict": { "type": "Cache", "ttl": 3600000, "namespacestore_dict": { "aws": [(1, "eu-central-1")], }, }, "placement_policy": { "tiers": [{ "backingStores": [constants.DEFAULT_NOOBAA_BACKINGSTORE] }] }, } cache_bucket = bucket_factory(bucketclass=cache_bucketclass)[0] cache_uploaded_objs_dir = uploaded_objects_dir + "/cache" cache_uploaded_objs_dir_2 = uploaded_objects_dir + "/cache_2" cache_downloaded_objs_dir = downloaded_obejcts_dir + "/cache" underlying_bucket_name = cache_bucket.bucketclass.namespacestores[ 0].uls_name # Upload a random object to the bucket objs_written_to_cache_bucket = write_random_test_objects_to_bucket( awscli_pod_session, cache_bucket.name, cache_uploaded_objs_dir, pattern=cache_pattern, mcg_obj=mcg_obj_session, ) wait_for_cache(mcg_obj_session, cache_bucket.name, objs_written_to_cache_bucket) # Write a random, larger object directly to the underlying storage of the bucket write_random_test_objects_to_bucket( awscli_pod_session, underlying_bucket_name, cache_uploaded_objs_dir_2, pattern=cache_pattern, s3_creds=cld_mgr.aws_client.nss_creds, ) # Download the object from the cache bucket sync_object_directory( awscli_pod_session, f"s3://{cache_bucket.name}", cache_downloaded_objs_dir, mcg_obj_session, ) # Make sure the cached object was returned, and not the one that was written to the underlying storage assert compare_directory( awscli_pod_session, cache_uploaded_objs_dir, cache_downloaded_objs_dir, amount=1, pattern=cache_pattern, ), "The uploaded and downloaded cached objects have different checksums" assert ( compare_directory( awscli_pod_session, cache_uploaded_objs_dir_2, cache_downloaded_objs_dir, amount=1, pattern=cache_pattern, ) is False ), "The cached object was replaced by the new one before the TTL has expired" return { "test_buckets": test_buckets, "test_buckets_upload_dir": uploaded_objects_dir, "object_amount": object_amount, "test_buckets_pattern": test_buckets_pattern, "first_bidi_bucket_name": first_bidi_bucket_name, "bidi_downloaded_objs_dir_2": bidi_downloaded_objs_dir_2, "first_bidirectional_pattern": first_bidirectional_pattern, "second_bidi_bucket_name": second_bidi_bucket_name, "second_bidirectional_pattern": second_bidirectional_pattern, "cache_bucket_name": cache_bucket.name, "cache_pattern": cache_pattern, "cache_downloaded_objs_dir": cache_downloaded_objs_dir, }