def test_send():
if os.path.isfile('/etc/cortx/cluster.conf'
) and os.stat("/etc/cortx/cluster.conf").st_size > 10:
s3iem = IEMutil("object_recovery_scheduler", "INFO", 100, "Test")
severity = 'X'
module = 'S3 BG delete'
event_id = 100
message_blob = "*******This is a test IEM message********"
result_data = s3iem.send(module, event_id, severity, message_blob)
assert result_data == True
else:
assert True == True
def test_receive():
if os.path.isfile('/etc/cortx/cluster.conf'
) and os.stat("/etc/cortx/cluster.conf").st_size > 10:
s3iem = IEMutil("object_recovery_scheduler", "INFO", 100, "Test")
component = 'S3'
severity = 'X'
module = 'S3 BG delete'
event_id = 100
message_blob = "Test message"
ret = s3iem.send(module, event_id, severity, message_blob)
result_data = s3iem.receive(component)
assert ret == True
assert result_data == True
else:
assert True == True
def head(self, oid, layout_id):
"""Perform HEAD request and generate response."""
if oid is None:
self._logger.error("Object Id is required.")
return False, None
if layout_id is None:
self._logger.error("Layout Id is required.")
return False, None
# The URL quoting functions focus on taking program data and making it safe for use as URL components by quoting special characters and appropriately encoding non-ASCII text.
# urllib.parse.urlencode converts a mapping object or a sequence of two-element tuples, which may contain str or bytes objects, to a percent-encoded ASCII text string.
# https://docs.python.org/3/library/urllib.parse.html
# For example if oid is 'JwZSAwAAAAA=-AgAAAAAA4Ag=' urllib.parse.quote(oid, safe='') yields 'JwZSAwAAAAA%3D-AgAAAAAA4Ag%3D' and layout_id is 1
# urllib.parse.urlencode({'layout-id': layout_id}) yields layout-id=1
# And request_uri is '/objects/JwZSAwAAAAA%3D-AgAAAAAA4Ag%3D' ,
# absolute_request_uri is layout-id is '/objects/JwZSAwAAAAA%3D-AgAAAAAA4Ag%3D?layout-id=1'
query_params = urllib.parse.urlencode({'layout-id': layout_id})
request_uri = '/objects/' + urllib.parse.quote(oid, safe='')
absolute_request_uri = request_uri + '?' + query_params
body = ''
headers = self.s3_util.prepare_signed_header('HEAD', request_uri,
query_params, body)
if headers['Authorization'] is None:
self._logger.error("Failed to generate v4 signature")
return False, None
try:
response = super(CORTXS3ObjectApi, self).head(absolute_request_uri,
body,
headers=headers)
except ConnectionRefusedError as ex:
IEMutil("ERROR", IEMutil.S3_CONN_FAILURE,
IEMutil.S3_CONN_FAILURE_STR)
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(502, "", "ConnectionRefused")
except Exception as ex:
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(500, "", "InternalServerError")
if response['status'] == 200:
self._logger.info("HEAD Object called successfully with status code: "\
+ str(response['status']) + " response body: " + str(response['body']))
return True, CORTXS3SuccessResponse(response['body'])
else:
self._logger.info("Failed to do HEAD Object with status code: "\
+ str(response['status']) + " response body: " + str(response['body']))
return False, CORTXS3ErrorResponse(response['status'],
response['reason'],
response['body'])
def put(self, index_id=None, object_key_name=None, value=""):
"""Perform PUT request and generate response."""
if index_id is None:
self._logger.error("Index Id is required.")
return False, None
if object_key_name is None:
self._logger.error("Key is required")
return False, None
query_params = ""
request_body = value
# The URL quoting functions focus on taking program data and making it safe for use as URL components by quoting special characters and appropriately encoding non-ASCII text.
# https://docs.python.org/3/library/urllib.parse.html
# For example if index_id is 'AAAAAAAAAHg=-AwAQAAAAAAA=' and object_key_name is "testobject+"
# urllib.parse.quote(index_id, safe='') and urllib.parse.quote(object_key_name) yields 'testobject%2B' respectively
# And request_uri is
# '/indexes/AAAAAAAAAHg%3D-AwAQAAAAAAA%3D/testobject%2B'
request_uri = '/indexes/' + \
urllib.parse.quote(index_id, safe='') + '/' + \
urllib.parse.quote(object_key_name)
headers = self.s3_util.prepare_signed_header('PUT', request_uri,
query_params,
request_body)
if (headers['Authorization'] is None):
self._logger.error("Failed to generate v4 signature")
return False, None
try:
response = super(CORTXS3KVApi, self).put(request_uri,
request_body,
headers=headers)
except ConnectionRefusedError as ex:
IEMutil("ERROR", IEMutil.S3_CONN_FAILURE,
IEMutil.S3_CONN_FAILURE_STR)
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(502, "", "ConnectionRefused")
except Exception as ex:
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(500, "", "InternalServerError")
if response['status'] == 200:
self._logger.info("Key value details added successfully.")
return True, CORTXS3SuccessResponse(response['body'])
else:
self._logger.info('Failed to add key value details.')
return False, CORTXS3ErrorResponse(response['status'],
response['reason'],
response['body'])
def put(self, oid, value):
"""Perform PUT request and generate response."""
if oid is None:
self._logger.error("Object Id is required.")
return False, None
query_params = ""
request_body = value
# The URL quoting functions focus on taking program data and making it safe for use as URL components by quoting special characters and appropriately encoding non-ASCII text.
# https://docs.python.org/3/library/urllib.parse.html
# For example if oid is 'JwZSAwAAAAA=-AgAAAAAA4Ag=' urllib.parse.quote(oid, safe='') yields 'JwZSAwAAAAA%3D-AgAAAAAA4Ag%3D'
# And request_uri is '/objects/JwZSAwAAAAA%3D-AgAAAAAA4Ag%3D'
request_uri = '/objects/' + urllib.parse.quote(oid, safe='')
headers = self.s3_util.prepare_signed_header('PUT', request_uri,
query_params,
request_body)
if headers['Authorization'] is None:
self._logger.error("Failed to generate v4 signature")
return False, None
try:
response = super(CORTXS3ObjectApi, self).put(request_uri,
request_body,
headers=headers)
except ConnectionRefusedError as ex:
IEMutil("ERROR", IEMutil.S3_CONN_FAILURE,
IEMutil.S3_CONN_FAILURE_STR)
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(502, "", "ConnectionRefused")
except Exception as ex:
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(500, "", "InternalServerError")
if response['status'] == 201:
self._logger.info("Object added successfully.")
return True, CORTXS3SuccessResponse(response['body'])
else:
self._logger.info('Failed to add Object.')
return False, CORTXS3ErrorResponse(response['status'],
response['reason'],
response['body'])
def delete(self, index_id):
"""Perform DELETE request and generate response."""
if index_id is None:
self._logger.info("Index Id is required.")
return False, None
# The URL quoting functions focus on taking program data and making it safe for use as URL components by quoting special characters and appropriately encoding non-ASCII text.
# https://docs.python.org/3/library/urllib.parse.html
# For example if index_id is 'AAAAAAAAAHg=-AwAQAAAAAAA=' urllib.parse.quote(index_id, safe='') yields 'AAAAAAAAAHg%3D-AwAQAAAAAAA%3D'
# And request_uri is '/indexes/AAAAAAAAAHg%3D-AwAQAAAAAAA%3D'
request_uri = '/indexes/' + urllib.parse.quote(index_id, safe='')
query_params = ""
body = ""
headers = self.s3_util.prepare_signed_header('DELETE', request_uri, query_params, body)
if(headers['Authorization'] is None):
self._logger.error("Failed to generate v4 signature")
return False, None
try:
response = super(
CORTXS3IndexApi,
self).delete(
request_uri,
headers=headers)
except ConnectionRefusedError as ex:
IEMutil("ERROR", IEMutil.S3_CONN_FAILURE, IEMutil.S3_CONN_FAILURE_STR)
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(502,"","ConnectionRefused")
except Exception as ex:
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(500,"","InternalServerError")
if response['status'] == 204:
self._logger.info('Successfully deleted Index.')
return True, CORTXS3SuccessResponse(response['body'])
else:
self._logger.info('Failed to delete Index.')
return False, CORTXS3ErrorResponse(
response['status'], response['reason'], response['body'])
def receive_data(self):
"""Receive data and create msg queue."""
try:
# Check if service is running in non-daemon mode
# then consumer should stop once queue is empty.
if not self.config.get_daemon_mode():
queue_state = self._channel.queue_declare(
queue=self._queue, durable=self._durable)
queue_msg_count = queue_state.method.message_count
self.worker(queue_msg_count)
return
else:
self._channel.queue_declare(queue=self._queue,
durable=self._durable)
self.worker()
except Exception as exception:
err_msg = "error:%s, %s" % (exception, traceback.format_exc())
IEMutil("ERROR", IEMutil.RABBIT_MQ_CONN_FAILURE,
IEMutil.RABBIT_MQ_CONN_FAILURE_STR)
self.logger.error("msg_queue receive data failed." + str(err_msg))
self.connect()
self.receive_data()
def list(self, index_id, max_keys=1000, next_marker=None, additional_Query_params=None):
"""Perform LIST request and generate response."""
if index_id is None:
self._logger.error("Index Id is required.")
return False, None
self._logger.info("Processing request in IndexAPI")
# The URL quoting functions focus on taking program data and making it safe for use as URL components by quoting special characters and appropriately encoding non-ASCII text.
# https://docs.python.org/3/library/urllib.parse.html
# For example if index_id is 'AAAAAAAAAHg=-AwAQAAAAAAA=' urllib.parse.quote(index_id, safe='') yields 'AAAAAAAAAHg%3D-AwAQAAAAAAA%3D'
# And request_uri is '/indexes/AAAAAAAAAHg%3D-AwAQAAAAAAA%3D'
request_uri = '/indexes/' + urllib.parse.quote(index_id, safe='')
query_params = ""
inputQueryParams = {}
inputQueryParams["keys"] = max_keys
if (next_marker is not None):
inputQueryParams["marker"] = next_marker
if (additional_Query_params is not None and isinstance(additional_Query_params, dict)):
# Add addtional query params
inputQueryParams.update(additional_Query_params)
#Generate sorted urlencoded query params into query_params
for key in sorted(inputQueryParams.keys()):
d = {}
d[key] = inputQueryParams[key]
if (query_params == ""):
query_params += urllib.parse.urlencode(d)
else:
query_params += "&" + urllib.parse.urlencode(d)
absolute_request_uri = request_uri + '?' + query_params
body = ""
headers = self.s3_util.prepare_signed_header('GET', request_uri, query_params, body)
if(headers['Authorization'] is None):
self._logger.error("Failed to generate v4 signature")
return False, None
try:
response = super(
CORTXS3IndexApi,
self).get(
absolute_request_uri,
headers=headers)
except ConnectionRefusedError as ex:
IEMutil("ERROR", IEMutil.S3_CONN_FAILURE, IEMutil.S3_CONN_FAILURE_STR)
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(502,"","ConnectionRefused")
except Exception as ex:
self._logger.error(repr(ex))
return False, CORTXS3ErrorResponse(500,"","InternalServerError")
if response['status'] == 200:
self._logger.info('Successfully listed Index details.')
return True, CORTXS3ListIndexResponse(response['body'])
else:
self._logger.info('Failed to list Index details.')
return False, CORTXS3ErrorResponse(
response['status'], response['reason'], response['body'])
def add_kv_to_queue(self, marker = None):
"""Add object key value to object recovery queue."""
self.logger.info("Adding kv list to queue")
try:
from s3backgrounddelete.object_recovery_queue import ObjectRecoveryRabbitMq
mq_client = ObjectRecoveryRabbitMq(
self.config,
self.config.get_rabbitmq_username(),
self.config.get_rabbitmq_password(),
self.config.get_rabbitmq_host(),
self.config.get_rabbitmq_exchange(),
self.config.get_rabbitmq_queue_name(),
self.config.get_rabbitmq_mode(),
self.config.get_rabbitmq_durable(),
self.logger)
# Cleanup all entries and enqueue only 1000 entries
mq_client.purge_queue(self.config.get_rabbitmq_queue_name())
result, index_response = CORTXS3IndexApi(
self.config, logger=self.logger).list(
self.config.get_probable_delete_index_id(), self.config.get_max_keys(), marker)
if result:
self.logger.info("Index listing result :" +
str(index_response.get_index_content()))
probable_delete_json = index_response.get_index_content()
probable_delete_oid_list = probable_delete_json["Keys"]
is_truncated = probable_delete_json["IsTruncated"]
if (probable_delete_oid_list is not None):
for record in probable_delete_oid_list:
# Check if record is older than the pre-configured 'time to process' delay
leak_processing_delay = self.config.get_leak_processing_delay_in_mins()
try:
objLeakVal = json.loads(record["Value"])
except ValueError as error:
self.logger.error(
"Failed to parse JSON data for: " + str(record) + " due to: " + error)
continue
if (objLeakVal is None):
self.logger.error("No value associated with " + str(record) + ". Skipping entry")
continue
# Check if object leak entry is older than 15mins or a preconfigured duration
if (not ObjectRecoveryScheduler.isObjectLeakEntryOlderThan(objLeakVal, leak_processing_delay)):
self.logger.info("Object leak entry " + record["Key"] +
" is NOT older than " + str(leak_processing_delay) +
"mins. Skipping entry")
continue
self.logger.info(
"Object recovery queue sending data :" +
str(record))
ret, msg = mq_client.send_data(
record, self.config.get_rabbitmq_queue_name())
if not ret:
IEMutil("ERROR", IEMutil.RABBIT_MQ_CONN_FAILURE, IEMutil.RABBIT_MQ_CONN_FAILURE_STR)
self.logger.error(
"Object recovery queue send data "+ str(record) +
" failed :" + msg)
else:
self.logger.info(
"Object recovery queue send data successfully :" +
str(record))
else:
self.logger.info(
"Index listing result empty. Ignoring adding entry to object recovery queue")
pass
else:
self.logger.error("Failed to retrive Index listing:")
except BaseException:
self.logger.error(
"Object recovery queue send data exception:" + traceback.format_exc())
finally:
if mq_client:
self.logger.info("Closing the mqclient")
mq_client.close()
def checkthreshold(self, thld, kvcount):
if(thld):
if kvcount > int(thld):
IEMutil("ERROR", IEMutil.THRESHOLD_CROSS, IEMutil.THRESHOLD_CROSS_STR)
print("The kv entries are more than threshold value")
sys.exit()