def test_list_no_index_id(): """Test List api without index_id should return response as "None".""" config = CORTXS3Config(use_cipher=False) response = CORTXS3IndexApi(config).list(None) if (response is not None): assert response[0] is False assert response[1] is None
def __init__(self, config, probable_delete_records, logger=None, objectapi=None, kvapi=None, indexapi=None): """Initialise Validator""" self.config = config self.current_obj_in_VersionList = None self.probable_delete_records = probable_delete_records if (logger is None): self._logger = logging.getLogger("ObjectRecoveryValidator") else: self._logger = logger if (objectapi is None): self._objectapi = CORTXS3ObjectApi(self.config, logger=self._logger) else: self._objectapi = objectapi if (kvapi is None): self._kvapi = CORTXS3KVApi(self.config, logger=self._logger) else: self._kvapi = kvapi if (indexapi is None): self._indexapi = CORTXS3IndexApi(self.config, logger=self._logger) else: self._indexapi = indexapi
def test_list_no_index_id(): """Test List api without index_id should return response as "None".""" config = CORTXS3Config() response = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER).list(None) if (response is not None): assert response[0] is False assert response[1] is None
def check_object_in_probable_delete_index(oid_dict): oid_str = str(list(oid_dict.keys())) oid_list = list(oid_dict.keys()) oid_deleted = False print("Probable delete list index should not contain: [" + oid_str + "]") config = CORTXS3Config() result, index_response = CORTXS3IndexApi( config, CONNECTION_TYPE_PRODUCER).list(config.get_probable_delete_index_id(), config.get_max_keys()) if result: if index_response is None: raise AssertionError probable_delete_json = index_response.get_index_content() probable_delete_oid_list = probable_delete_json["Keys"] if (probable_delete_oid_list is not None and len(probable_delete_oid_list) > 0): print( "Probable delete list is non-empty. Checking presence of oid:[" + oid_str + "] ...") if any(oid in probable_delete_oid_list for oid in oid_list): print("Probable delete list contains oid [" + oid_str + "]. Expected no such oid") raise AssertionError else: oid_deleted = True else: print("Probable delete list is empty") oid_deleted = True else: # Something failed badly raise AssertionError if (oid_deleted): print("Success - Object oid [" + oid_str + "] is not present in list")
def count(self, index_id="AAAAAAAAAHg=-AwAQAAAAAAA=", threshold=None): CONFIG = CORTXS3Config() index_api = CORTXS3IndexApi(CONFIG, connectionType=CONNECTION_TYPE_PRODUCER) response, data = index_api.list(index_id) if (response): if isinstance(data, CORTXS3ListIndexResponse): list_index_response = data.get_index_content() total = self.itercount(list_index_response) self.checkthreshold(threshold, total) print(f'Iteration count0 : {total}') is_truncated = list_index_response["IsTruncated"] itr = 0 while (is_truncated == "true"): response, data = index_api.list( index_id, max_keys=1000, next_marker=list_index_response["NextMarker"]) if isinstance(data, CORTXS3ListIndexResponse): list_index_response = data.get_index_content() total = total + self.itercount(list_index_response) self.checkthreshold(threshold, total) is_truncated = list_index_response["IsTruncated"] elif isinstance(data, CORTXS3ErrorResponse): print(data.get_error_message()) break itr = itr + 1 print( f'Iteration count{itr} : {self.itercount(list_index_response)}' ) print(f'Total Count of kv entries : {total}') else: print("Error") print(data.get_error_message())
def test_put_failure(): """Test if index is already present then PUT should return failure response.""" httpconnection = Mock(spec=HTTPConnection) httpresponse = Mock(spec=HTTPResponse) httpresponse.status = 409 httpresponse.getheaders.return_value = \ 'Content-Type:text/html;Content-Length:14' httpresponse.read.return_value = b'{}' httpresponse.reason = 'CONFLICT' httpconnection.getresponse.return_value = httpresponse config = CORTXS3Config() response = CORTXS3IndexApi(config, connection=httpconnection).put("test_index1") if (response is not None): assert response[0] is False
def test_put_success(): """Test PUT api, it should return success response.""" httpconnection = Mock(spec=HTTPConnection) httpresponse = Mock(spec=HTTPResponse) httpresponse.status = 201 httpresponse.getheaders.return_value = \ 'Content-Type:text/html;Content-Length:14' httpresponse.read.return_value = b'{}' httpresponse.reason = 'CREATED' httpconnection.getresponse.return_value = httpresponse config = CORTXS3Config() response = CORTXS3IndexApi(config, connection=httpconnection).put("test_index1") if (response is not None): assert response[0] is True
def test_list_failure(): """Test if index is not present then List should return failure response.""" httpconnection = Mock(spec=HTTPConnection) httpresponse = Mock(spec=HTTPResponse) httpresponse.status = 404 httpresponse.getheaders.return_value = \ 'Content-Type:text/html;Content-Length:14' httpresponse.read.return_value = b'{}' httpresponse.reason = 'NOT FOUND' httpconnection.getresponse.return_value = httpresponse config = CORTXS3Config() response = CORTXS3IndexApi(config, connection=httpconnection).list("test_index2") if (response is not None): assert response[0] is False assert response[1] is not None
def test_list_success(): """Test List api, it should return success response.""" result = b'{"index_id":"test_index1","object_layout_id":1,"object_metadata_path":"test_index1"}' httpconnection = Mock(spec=HTTPConnection) httpresponse = Mock(spec=HTTPResponse) httpresponse.status = 200 httpresponse.getheaders.return_value = \ 'Content-Type:text/html;Content-Length:14' httpresponse.read.return_value = result httpresponse.reason = 'OK' httpconnection.getresponse.return_value = httpresponse config = CORTXS3Config() response = CORTXS3IndexApi(config, connection=httpconnection).list("test_index1") if (response is not None): assert response[0] is True assert response[1] is not None
def assert_index(index_name, index_id, expected_kv_count=0, error_message=None): print("Checking {}".format(index_name)) index_api = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER) # Instance index has startup entries, other indexes should be empty. max_keys = 1 if expected_kv_count > 0: max_keys = expected_kv_count response, data = index_api.list(index_id, max_keys) if response: list_index_response = data.get_index_content() keys = list_index_response['Keys'] if expected_kv_count == 0: if keys is not None: print("Index {} is not empty".format(index_name)) if error_message is not None: print(error_message) sys.exit(1) else: if len(keys) != expected_kv_count: print("Index {} does not have expected entries".format( index_name)) if error_message is not None: print(error_message) sys.exit(1) print("Success") # Index listing failed. S3server might be down. else: print("Error while listing index {}".format(index_name)) sys.exit(1)
# limitations under the License. # # For any questions about this software or licensing, # please email [email protected] or [email protected]. """ To List KV for a index_id you need to execute this file as: python36 list_kv.py index_id key it's like python36 list_kv.py sys.argv[1] """ import sys from s3backgrounddelete.cortx_s3_config import CORTXS3Config from s3backgrounddelete.cortx_s3_index_api import CORTXS3IndexApi from s3backgrounddelete.cortx_s3_constants import CONNECTION_TYPE_PRODUCER CONFIG = CORTXS3Config() index_api = CORTXS3IndexApi(CONFIG, CONNECTION_TYPE_PRODUCER) response, data = index_api.list(sys.argv[1]) if(response): list_index_response = data.get_index_content() print(str(list_index_response)) is_truncated = list_index_response["IsTruncated"] while(is_truncated == "true"): response, data = index_api.list(sys.argv[1], list_index_response["NextMarker"]) list_index_response = data.get_index_content() is_truncated = list_index_response["IsTruncated"] print(str(list_index_response)) else: print("Error") print(data.get_error_message())
print("Running Processor...") processor.consume() print("Processor has stopped...") # ************* Verify clean up of OID's***************** check_object_in_probable_delete_index(multipart_oid_dict) # ************* Verify cleanup of Object using aws s3api head-object api***************** AwsTest('Do head-object for "object8" on bucket "seagatebucket"')\ .head_object("seagatebucket", "object8").execute_test(negative_case=True)\ .command_should_fail().command_error_should_have("Not Found") # ************* Verify part list index is deleted ************* # Use HEAD /indexes/<index oid> API to ensure that # the part list index is deleted by the object leak task. status, res = CORTXS3IndexApi(CONFIG, CONNECTION_TYPE_PRODUCER).head(part_index) if (not status): if (res): assert isinstance(res, CORTXS3ErrorResponse) assert res.get_error_status() == 404 assert res.get_error_reason() == "Not Found" print("Index id \"" + part_index + "\" does not exist") print("Part index deleted") else: assert False, "Error: Unable to verify part index leak" """ Test Scenario : 8 Scenario: Delayed delete object test (DELETE api test) 1. create object and the get OID, layout_id from response 2. delete object 3. run background delete schedular
def __init__(self): self.config = CORTXS3Config(s3recovery_flag=True) self.index_api = CORTXS3IndexApi(self.config) self.kv_api = CORTXS3KVApi(self.config) self.log_result = False self.logger = None
# See the License for the specific language governing permissions and # limitations under the License. # # For any questions about this software or licensing, # please email [email protected] or [email protected]. """ To List KV for a index_id you need to execute this file as: python36 list_kv.py index_id key it's like python36 list_kv.py sys.argv[1] """ import sys from s3backgrounddelete.cortx_s3_config import CORTXS3Config from s3backgrounddelete.cortx_s3_index_api import CORTXS3IndexApi CONFIG = CORTXS3Config() index_api = CORTXS3IndexApi(CONFIG) response, data = index_api.list(sys.argv[1]) if (response): list_index_response = data.get_index_content() print(str(list_index_response)) is_truncated = list_index_response["IsTruncated"] while (is_truncated == "true"): response, data = index_api.list(sys.argv[1], list_index_response["NextMarker"]) list_index_response = data.get_index_content() is_truncated = list_index_response["IsTruncated"] print(str(list_index_response)) else: print("Error") print(data.get_error_message())
def test_put_no_index_id(): """Test PUT request without index_id, it should return response as "None".""" config = CORTXS3Config(use_cipher=False) response = CORTXS3IndexApi(config).put(None) if (response is not None): assert response[0] is False
def add_kv_to_queue(self, marker = None): """Add object key value to object recovery queue.""" self.logger.info("Adding kv list to queue") try: from s3backgrounddelete.object_recovery_queue import ObjectRecoveryRabbitMq mq_client = ObjectRecoveryRabbitMq( self.config, self.config.get_rabbitmq_username(), self.config.get_rabbitmq_password(), self.config.get_rabbitmq_host(), self.config.get_rabbitmq_exchange(), self.config.get_rabbitmq_queue_name(), self.config.get_rabbitmq_mode(), self.config.get_rabbitmq_durable(), self.logger) # Cleanup all entries and enqueue only 1000 entries mq_client.purge_queue(self.config.get_rabbitmq_queue_name()) result, index_response = CORTXS3IndexApi( self.config, logger=self.logger).list( self.config.get_probable_delete_index_id(), self.config.get_max_keys(), marker) if result: self.logger.info("Index listing result :" + str(index_response.get_index_content())) probable_delete_json = index_response.get_index_content() probable_delete_oid_list = probable_delete_json["Keys"] is_truncated = probable_delete_json["IsTruncated"] if (probable_delete_oid_list is not None): for record in probable_delete_oid_list: # Check if record is older than the pre-configured 'time to process' delay leak_processing_delay = self.config.get_leak_processing_delay_in_mins() try: objLeakVal = json.loads(record["Value"]) except ValueError as error: self.logger.error( "Failed to parse JSON data for: " + str(record) + " due to: " + error) continue if (objLeakVal is None): self.logger.error("No value associated with " + str(record) + ". Skipping entry") continue # Check if object leak entry is older than 15mins or a preconfigured duration if (not ObjectRecoveryScheduler.isObjectLeakEntryOlderThan(objLeakVal, leak_processing_delay)): self.logger.info("Object leak entry " + record["Key"] + " is NOT older than " + str(leak_processing_delay) + "mins. Skipping entry") continue self.logger.info( "Object recovery queue sending data :" + str(record)) ret, msg = mq_client.send_data( record, self.config.get_rabbitmq_queue_name()) if not ret: IEMutil("ERROR", IEMutil.RABBIT_MQ_CONN_FAILURE, IEMutil.RABBIT_MQ_CONN_FAILURE_STR) self.logger.error( "Object recovery queue send data "+ str(record) + " failed :" + msg) else: self.logger.info( "Object recovery queue send data successfully :" + str(record)) else: self.logger.info( "Index listing result empty. Ignoring adding entry to object recovery queue") pass else: self.logger.error("Failed to retrive Index listing:") except BaseException: self.logger.error( "Object recovery queue send data exception:" + traceback.format_exc()) finally: if mq_client: self.logger.info("Closing the mqclient") mq_client.close()
print("Running Processor...") processor.consume() print("Processor has stopped...") # ************* Verify clean up of OID's***************** perform_head_object(multipart_oid_dict) # ************* Verify cleanup of Object using aws s3api head-object api***************** AwsTest('Do head-object for "object8" on bucket "seagatebucket"')\ .head_object("seagatebucket", "object8").execute_test(negative_case=True)\ .command_should_fail().command_error_should_have("Not Found") # ************* Verify part list index is deleted ************* # Use HEAD /indexes/<index oid> API to ensure that # the part list index is deleted by the object leak task. status, res = CORTXS3IndexApi(CONFIG).head(part_index) if (not status): if (res): assert isinstance(res, CORTXS3ErrorResponse) assert res.get_error_status() == 404 assert res.get_error_reason() == "Not Found" print("Index id \"" + part_index + "\" does not exist") print("Part index deleted") else: assert False, "Error: Unable to verify part index leak" # ****** Delete bucket "seagatebucket" using s3-background-delete-svc account***** AwsTest('Delete Bucket "seagatebucket"').delete_bucket("seagatebucket")\ .execute_test().command_is_successful() #Clear probable delete list index
def add_kv_to_msgbus(self, marker=None): """Add object key value to msgbus topic.""" self.logger.info("Inside add_kv_to_msgbus.") try: from s3backgrounddelete.object_recovery_msgbus import ObjectRecoveryMsgbus if not self.producer: self.producer = ObjectRecoveryMsgbus(self.config, self.logger) threshold = self.config.get_threshold() self.logger.debug("Threshold is : " + str(threshold)) count = self.producer.get_count() self.logger.debug("Count of unread msgs is : " + str(count)) if ((int(count) < threshold) or (threshold == 0)): self.logger.debug( "Count of unread messages is less than threshold value.Hence continuing..." ) else: #do nothing self.logger.info( "Queue has more messages than threshold value. Hence skipping addition of further entries." ) return # Cleanup all entries and enqueue only 1000 entries #PurgeAPI Here self.producer.purge() result, index_response = CORTXS3IndexApi( self.config, connectionType=CONNECTION_TYPE_PRODUCER, logger=self.logger).list( self.config.get_probable_delete_index_id(), self.config.get_max_keys(), marker) if result: self.logger.info("Index listing result :" + str(index_response.get_index_content())) probable_delete_json = index_response.get_index_content() probable_delete_oid_list = probable_delete_json["Keys"] is_truncated = probable_delete_json["IsTruncated"] if (probable_delete_oid_list is not None): for record in probable_delete_oid_list: # Check if record is older than the pre-configured 'time to process' delay leak_processing_delay = self.config.get_leak_processing_delay_in_mins( ) try: objLeakVal = json.loads(record["Value"]) except ValueError as error: self.logger.error( "Failed to parse JSON data for: " + str(record) + " due to: " + error) continue if (objLeakVal is None): self.logger.error("No value associated with " + str(record) + ". Skipping entry") continue # Check if object leak entry is older than 15mins or a preconfigured duration if (not ObjectRecoveryScheduler. isObjectLeakEntryOlderThan( objLeakVal, leak_processing_delay)): self.logger.info("Object leak entry " + record["Key"] + " is NOT older than " + str(leak_processing_delay) + "mins. Skipping entry") continue self.logger.info( "Object recovery queue sending data :" + str(record)) ret = self.producer.send_data( record, producer_id=self.producer_name) if not ret: # TODO - Do Audit logging self.logger.error( "Object recovery queue send data " + str(record) + " failed :") else: self.logger.info( "Object recovery queue send data successfully :" + str(record)) else: self.logger.info( "Index listing result empty. Ignoring adding entry to object recovery queue" ) else: self.logger.error("Failed to retrive Index listing:") except Exception as exception: self.logger.error( "add_kv_to_msgbus send data exception: {}".format(exception)) self.logger.debug("traceback : {}".format(traceback.format_exc()))
def test_put_no_index_id(): """Test PUT request without index_id, it should return response as "None".""" config = CORTXS3Config() response = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER).put(None) if (response is not None): assert response[0] is False