def test_list_no_index_id():
"""Test List api without index_id should return response as "None"."""
config = CORTXS3Config(use_cipher=False)
response = CORTXS3IndexApi(config).list(None)
if (response is not None):
assert response[0] is False
assert response[1] is None
def __init__(self,
config,
probable_delete_records,
logger=None,
objectapi=None,
kvapi=None,
indexapi=None):
"""Initialise Validator"""
self.config = config
self.current_obj_in_VersionList = None
self.probable_delete_records = probable_delete_records
if (logger is None):
self._logger = logging.getLogger("ObjectRecoveryValidator")
else:
self._logger = logger
if (objectapi is None):
self._objectapi = CORTXS3ObjectApi(self.config,
logger=self._logger)
else:
self._objectapi = objectapi
if (kvapi is None):
self._kvapi = CORTXS3KVApi(self.config, logger=self._logger)
else:
self._kvapi = kvapi
if (indexapi is None):
self._indexapi = CORTXS3IndexApi(self.config, logger=self._logger)
else:
self._indexapi = indexapi
def test_list_no_index_id():
"""Test List api without index_id should return response as "None"."""
config = CORTXS3Config()
response = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER).list(None)
if (response is not None):
assert response[0] is False
assert response[1] is None
def check_object_in_probable_delete_index(oid_dict):
oid_str = str(list(oid_dict.keys()))
oid_list = list(oid_dict.keys())
oid_deleted = False
print("Probable delete list index should not contain: [" + oid_str + "]")
config = CORTXS3Config()
result, index_response = CORTXS3IndexApi(
config,
CONNECTION_TYPE_PRODUCER).list(config.get_probable_delete_index_id(),
config.get_max_keys())
if result:
if index_response is None:
raise AssertionError
probable_delete_json = index_response.get_index_content()
probable_delete_oid_list = probable_delete_json["Keys"]
if (probable_delete_oid_list is not None
and len(probable_delete_oid_list) > 0):
print(
"Probable delete list is non-empty. Checking presence of oid:["
+ oid_str + "] ...")
if any(oid in probable_delete_oid_list for oid in oid_list):
print("Probable delete list contains oid [" + oid_str +
"]. Expected no such oid")
raise AssertionError
else:
oid_deleted = True
else:
print("Probable delete list is empty")
oid_deleted = True
else:
# Something failed badly
raise AssertionError
if (oid_deleted):
print("Success - Object oid [" + oid_str + "] is not present in list")
def count(self, index_id="AAAAAAAAAHg=-AwAQAAAAAAA=", threshold=None):
CONFIG = CORTXS3Config()
index_api = CORTXS3IndexApi(CONFIG,
connectionType=CONNECTION_TYPE_PRODUCER)
response, data = index_api.list(index_id)
if (response):
if isinstance(data, CORTXS3ListIndexResponse):
list_index_response = data.get_index_content()
total = self.itercount(list_index_response)
self.checkthreshold(threshold, total)
print(f'Iteration count0 : {total}')
is_truncated = list_index_response["IsTruncated"]
itr = 0
while (is_truncated == "true"):
response, data = index_api.list(
index_id,
max_keys=1000,
next_marker=list_index_response["NextMarker"])
if isinstance(data, CORTXS3ListIndexResponse):
list_index_response = data.get_index_content()
total = total + self.itercount(list_index_response)
self.checkthreshold(threshold, total)
is_truncated = list_index_response["IsTruncated"]
elif isinstance(data, CORTXS3ErrorResponse):
print(data.get_error_message())
break
itr = itr + 1
print(
f'Iteration count{itr} : {self.itercount(list_index_response)}'
)
print(f'Total Count of kv entries : {total}')
else:
print("Error")
print(data.get_error_message())
def test_put_failure():
"""Test if index is already present then PUT should return failure response."""
httpconnection = Mock(spec=HTTPConnection)
httpresponse = Mock(spec=HTTPResponse)
httpresponse.status = 409
httpresponse.getheaders.return_value = \
'Content-Type:text/html;Content-Length:14'
httpresponse.read.return_value = b'{}'
httpresponse.reason = 'CONFLICT'
httpconnection.getresponse.return_value = httpresponse
config = CORTXS3Config()
response = CORTXS3IndexApi(config, connection=httpconnection).put("test_index1")
if (response is not None):
assert response[0] is False
def test_put_success():
"""Test PUT api, it should return success response."""
httpconnection = Mock(spec=HTTPConnection)
httpresponse = Mock(spec=HTTPResponse)
httpresponse.status = 201
httpresponse.getheaders.return_value = \
'Content-Type:text/html;Content-Length:14'
httpresponse.read.return_value = b'{}'
httpresponse.reason = 'CREATED'
httpconnection.getresponse.return_value = httpresponse
config = CORTXS3Config()
response = CORTXS3IndexApi(config, connection=httpconnection).put("test_index1")
if (response is not None):
assert response[0] is True
def test_list_failure():
"""Test if index is not present then List should return failure response."""
httpconnection = Mock(spec=HTTPConnection)
httpresponse = Mock(spec=HTTPResponse)
httpresponse.status = 404
httpresponse.getheaders.return_value = \
'Content-Type:text/html;Content-Length:14'
httpresponse.read.return_value = b'{}'
httpresponse.reason = 'NOT FOUND'
httpconnection.getresponse.return_value = httpresponse
config = CORTXS3Config()
response = CORTXS3IndexApi(config, connection=httpconnection).list("test_index2")
if (response is not None):
assert response[0] is False
assert response[1] is not None
def test_list_success():
"""Test List api, it should return success response."""
result = b'{"index_id":"test_index1","object_layout_id":1,"object_metadata_path":"test_index1"}'
httpconnection = Mock(spec=HTTPConnection)
httpresponse = Mock(spec=HTTPResponse)
httpresponse.status = 200
httpresponse.getheaders.return_value = \
'Content-Type:text/html;Content-Length:14'
httpresponse.read.return_value = result
httpresponse.reason = 'OK'
httpconnection.getresponse.return_value = httpresponse
config = CORTXS3Config()
response = CORTXS3IndexApi(config, connection=httpconnection).list("test_index1")
if (response is not None):
assert response[0] is True
assert response[1] is not None
def assert_index(index_name,
index_id,
expected_kv_count=0,
error_message=None):
print("Checking {}".format(index_name))
index_api = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER)
# Instance index has startup entries, other indexes should be empty.
max_keys = 1
if expected_kv_count > 0:
max_keys = expected_kv_count
response, data = index_api.list(index_id, max_keys)
if response:
list_index_response = data.get_index_content()
keys = list_index_response['Keys']
if expected_kv_count == 0:
if keys is not None:
print("Index {} is not empty".format(index_name))
if error_message is not None:
print(error_message)
sys.exit(1)
else:
if len(keys) != expected_kv_count:
print("Index {} does not have expected entries".format(
index_name))
if error_message is not None:
print(error_message)
sys.exit(1)
print("Success")
# Index listing failed. S3server might be down.
else:
print("Error while listing index {}".format(index_name))
sys.exit(1)
# limitations under the License. # # For any questions about this software or licensing, # please email [email protected] or [email protected]. """ To List KV for a index_id you need to execute this file as: python36 list_kv.py index_id key it's like python36 list_kv.py sys.argv[1] """ import sys from s3backgrounddelete.cortx_s3_config import CORTXS3Config from s3backgrounddelete.cortx_s3_index_api import CORTXS3IndexApi from s3backgrounddelete.cortx_s3_constants import CONNECTION_TYPE_PRODUCER CONFIG = CORTXS3Config() index_api = CORTXS3IndexApi(CONFIG, CONNECTION_TYPE_PRODUCER) response, data = index_api.list(sys.argv[1]) if(response): list_index_response = data.get_index_content() print(str(list_index_response)) is_truncated = list_index_response["IsTruncated"] while(is_truncated == "true"): response, data = index_api.list(sys.argv[1], list_index_response["NextMarker"]) list_index_response = data.get_index_content() is_truncated = list_index_response["IsTruncated"] print(str(list_index_response)) else: print("Error") print(data.get_error_message())
print("Running Processor...")
processor.consume()
print("Processor has stopped...")
# ************* Verify clean up of OID's*****************
check_object_in_probable_delete_index(multipart_oid_dict)
# ************* Verify cleanup of Object using aws s3api head-object api*****************
AwsTest('Do head-object for "object8" on bucket "seagatebucket"')\
.head_object("seagatebucket", "object8").execute_test(negative_case=True)\
.command_should_fail().command_error_should_have("Not Found")
# ************* Verify part list index is deleted *************
# Use HEAD /indexes/<index oid> API to ensure that
# the part list index is deleted by the object leak task.
status, res = CORTXS3IndexApi(CONFIG,
CONNECTION_TYPE_PRODUCER).head(part_index)
if (not status):
if (res):
assert isinstance(res, CORTXS3ErrorResponse)
assert res.get_error_status() == 404
assert res.get_error_reason() == "Not Found"
print("Index id \"" + part_index + "\" does not exist")
print("Part index deleted")
else:
assert False, "Error: Unable to verify part index leak"
"""
Test Scenario : 8
Scenario: Delayed delete object test (DELETE api test)
1. create object and the get OID, layout_id from response
2. delete object
3. run background delete schedular
def __init__(self):
self.config = CORTXS3Config(s3recovery_flag=True)
self.index_api = CORTXS3IndexApi(self.config)
self.kv_api = CORTXS3KVApi(self.config)
self.log_result = False
self.logger = None
# See the License for the specific language governing permissions and # limitations under the License. # # For any questions about this software or licensing, # please email [email protected] or [email protected]. """ To List KV for a index_id you need to execute this file as: python36 list_kv.py index_id key it's like python36 list_kv.py sys.argv[1] """ import sys from s3backgrounddelete.cortx_s3_config import CORTXS3Config from s3backgrounddelete.cortx_s3_index_api import CORTXS3IndexApi CONFIG = CORTXS3Config() index_api = CORTXS3IndexApi(CONFIG) response, data = index_api.list(sys.argv[1]) if (response): list_index_response = data.get_index_content() print(str(list_index_response)) is_truncated = list_index_response["IsTruncated"] while (is_truncated == "true"): response, data = index_api.list(sys.argv[1], list_index_response["NextMarker"]) list_index_response = data.get_index_content() is_truncated = list_index_response["IsTruncated"] print(str(list_index_response)) else: print("Error") print(data.get_error_message())
def test_put_no_index_id():
"""Test PUT request without index_id, it should return response as "None"."""
config = CORTXS3Config(use_cipher=False)
response = CORTXS3IndexApi(config).put(None)
if (response is not None):
assert response[0] is False
def add_kv_to_queue(self, marker = None):
"""Add object key value to object recovery queue."""
self.logger.info("Adding kv list to queue")
try:
from s3backgrounddelete.object_recovery_queue import ObjectRecoveryRabbitMq
mq_client = ObjectRecoveryRabbitMq(
self.config,
self.config.get_rabbitmq_username(),
self.config.get_rabbitmq_password(),
self.config.get_rabbitmq_host(),
self.config.get_rabbitmq_exchange(),
self.config.get_rabbitmq_queue_name(),
self.config.get_rabbitmq_mode(),
self.config.get_rabbitmq_durable(),
self.logger)
# Cleanup all entries and enqueue only 1000 entries
mq_client.purge_queue(self.config.get_rabbitmq_queue_name())
result, index_response = CORTXS3IndexApi(
self.config, logger=self.logger).list(
self.config.get_probable_delete_index_id(), self.config.get_max_keys(), marker)
if result:
self.logger.info("Index listing result :" +
str(index_response.get_index_content()))
probable_delete_json = index_response.get_index_content()
probable_delete_oid_list = probable_delete_json["Keys"]
is_truncated = probable_delete_json["IsTruncated"]
if (probable_delete_oid_list is not None):
for record in probable_delete_oid_list:
# Check if record is older than the pre-configured 'time to process' delay
leak_processing_delay = self.config.get_leak_processing_delay_in_mins()
try:
objLeakVal = json.loads(record["Value"])
except ValueError as error:
self.logger.error(
"Failed to parse JSON data for: " + str(record) + " due to: " + error)
continue
if (objLeakVal is None):
self.logger.error("No value associated with " + str(record) + ". Skipping entry")
continue
# Check if object leak entry is older than 15mins or a preconfigured duration
if (not ObjectRecoveryScheduler.isObjectLeakEntryOlderThan(objLeakVal, leak_processing_delay)):
self.logger.info("Object leak entry " + record["Key"] +
" is NOT older than " + str(leak_processing_delay) +
"mins. Skipping entry")
continue
self.logger.info(
"Object recovery queue sending data :" +
str(record))
ret, msg = mq_client.send_data(
record, self.config.get_rabbitmq_queue_name())
if not ret:
IEMutil("ERROR", IEMutil.RABBIT_MQ_CONN_FAILURE, IEMutil.RABBIT_MQ_CONN_FAILURE_STR)
self.logger.error(
"Object recovery queue send data "+ str(record) +
" failed :" + msg)
else:
self.logger.info(
"Object recovery queue send data successfully :" +
str(record))
else:
self.logger.info(
"Index listing result empty. Ignoring adding entry to object recovery queue")
pass
else:
self.logger.error("Failed to retrive Index listing:")
except BaseException:
self.logger.error(
"Object recovery queue send data exception:" + traceback.format_exc())
finally:
if mq_client:
self.logger.info("Closing the mqclient")
mq_client.close()
print("Running Processor...")
processor.consume()
print("Processor has stopped...")
# ************* Verify clean up of OID's*****************
perform_head_object(multipart_oid_dict)
# ************* Verify cleanup of Object using aws s3api head-object api*****************
AwsTest('Do head-object for "object8" on bucket "seagatebucket"')\
.head_object("seagatebucket", "object8").execute_test(negative_case=True)\
.command_should_fail().command_error_should_have("Not Found")
# ************* Verify part list index is deleted *************
# Use HEAD /indexes/<index oid> API to ensure that
# the part list index is deleted by the object leak task.
status, res = CORTXS3IndexApi(CONFIG).head(part_index)
if (not status):
if (res):
assert isinstance(res, CORTXS3ErrorResponse)
assert res.get_error_status() == 404
assert res.get_error_reason() == "Not Found"
print("Index id \"" + part_index + "\" does not exist")
print("Part index deleted")
else:
assert False, "Error: Unable to verify part index leak"
# ****** Delete bucket "seagatebucket" using s3-background-delete-svc account*****
AwsTest('Delete Bucket "seagatebucket"').delete_bucket("seagatebucket")\
.execute_test().command_is_successful()
#Clear probable delete list index
def add_kv_to_msgbus(self, marker=None):
"""Add object key value to msgbus topic."""
self.logger.info("Inside add_kv_to_msgbus.")
try:
from s3backgrounddelete.object_recovery_msgbus import ObjectRecoveryMsgbus
if not self.producer:
self.producer = ObjectRecoveryMsgbus(self.config, self.logger)
threshold = self.config.get_threshold()
self.logger.debug("Threshold is : " + str(threshold))
count = self.producer.get_count()
self.logger.debug("Count of unread msgs is : " + str(count))
if ((int(count) < threshold) or (threshold == 0)):
self.logger.debug(
"Count of unread messages is less than threshold value.Hence continuing..."
)
else:
#do nothing
self.logger.info(
"Queue has more messages than threshold value. Hence skipping addition of further entries."
)
return
# Cleanup all entries and enqueue only 1000 entries
#PurgeAPI Here
self.producer.purge()
result, index_response = CORTXS3IndexApi(
self.config,
connectionType=CONNECTION_TYPE_PRODUCER,
logger=self.logger).list(
self.config.get_probable_delete_index_id(),
self.config.get_max_keys(), marker)
if result:
self.logger.info("Index listing result :" +
str(index_response.get_index_content()))
probable_delete_json = index_response.get_index_content()
probable_delete_oid_list = probable_delete_json["Keys"]
is_truncated = probable_delete_json["IsTruncated"]
if (probable_delete_oid_list is not None):
for record in probable_delete_oid_list:
# Check if record is older than the pre-configured 'time to process' delay
leak_processing_delay = self.config.get_leak_processing_delay_in_mins(
)
try:
objLeakVal = json.loads(record["Value"])
except ValueError as error:
self.logger.error(
"Failed to parse JSON data for: " +
str(record) + " due to: " + error)
continue
if (objLeakVal is None):
self.logger.error("No value associated with " +
str(record) + ". Skipping entry")
continue
# Check if object leak entry is older than 15mins or a preconfigured duration
if (not ObjectRecoveryScheduler.
isObjectLeakEntryOlderThan(
objLeakVal, leak_processing_delay)):
self.logger.info("Object leak entry " +
record["Key"] +
" is NOT older than " +
str(leak_processing_delay) +
"mins. Skipping entry")
continue
self.logger.info(
"Object recovery queue sending data :" +
str(record))
ret = self.producer.send_data(
record, producer_id=self.producer_name)
if not ret:
# TODO - Do Audit logging
self.logger.error(
"Object recovery queue send data " +
str(record) + " failed :")
else:
self.logger.info(
"Object recovery queue send data successfully :"
+ str(record))
else:
self.logger.info(
"Index listing result empty. Ignoring adding entry to object recovery queue"
)
else:
self.logger.error("Failed to retrive Index listing:")
except Exception as exception:
self.logger.error(
"add_kv_to_msgbus send data exception: {}".format(exception))
self.logger.debug("traceback : {}".format(traceback.format_exc()))
def test_put_no_index_id():
"""Test PUT request without index_id, it should return response as "None"."""
config = CORTXS3Config()
response = CORTXS3IndexApi(config, CONNECTION_TYPE_PRODUCER).put(None)
if (response is not None):
assert response[0] is False