class ObjectRecoveryProcessor(object):
"""Provides consumer for object recovery"""
def __init__(self):
"""Initialise Server, config and create logger."""
self.server = None
self.config = CORTXS3Config()
self.create_logger_directory()
self.create_logger()
self.signal = DynamicConfigHandler(self)
self.logger.info("Initialising the Object Recovery Processor")
def consume(self):
"""Consume the objects from object recovery queue."""
self.server = None
try:
self.server = ObjectRecoveryRabbitMq(
self.config, self.config.get_rabbitmq_username(),
self.config.get_rabbitmq_password(),
self.config.get_rabbitmq_host(),
self.config.get_rabbitmq_exchange(),
self.config.get_rabbitmq_queue_name(),
self.config.get_rabbitmq_mode(),
self.config.get_rabbitmq_durable(), self.logger)
self.logger.info("Consumer started at " +
str(datetime.datetime.now()))
self.server.receive_data()
except BaseException:
if self.server:
self.server.close()
self.logger.error("main except:" + str(traceback.format_exc()))
def create_logger(self):
"""Create logger, file handler, formatter."""
# Create logger with "object_recovery_processor"
self.logger = logging.getLogger(
self.config.get_processor_logger_name())
self.logger.setLevel(self.config.get_file_log_level())
# create file handler which logs even debug messages
fhandler = logging.handlers.RotatingFileHandler(
self.config.get_processor_logger_file(),
mode='a',
maxBytes=self.config.get_max_bytes(),
backupCount=self.config.get_backup_count(),
encoding=None,
delay=False)
fhandler.setLevel(self.config.get_file_log_level())
# create console handler with a higher log level
chandler = logging.StreamHandler()
chandler.setLevel(self.config.get_console_log_level())
# create formatter and add it to the handlers
formatter = logging.Formatter(self.config.get_log_format())
fhandler.setFormatter(formatter)
chandler.setFormatter(formatter)
# add the handlers to the logger
self.logger.addHandler(fhandler)
self.logger.addHandler(chandler)
def close(self):
"""Stop processor and close rabbitmq connection."""
self.logger.info("Stopping the processor and rabbitmq connection")
self.server.close()
# perform an orderly shutdown by flushing and closing all handlers
logging.shutdown()
def create_logger_directory(self):
"""Create log directory if not exsists."""
self._logger_directory = os.path.join(
self.config.get_logger_directory())
if not os.path.isdir(self._logger_directory):
try:
os.mkdir(self._logger_directory)
except BaseException:
self.logger.error("Unable to create log directory at " +
self._logger_directory)
def add_kv_to_queue(self, marker = None):
"""Add object key value to object recovery queue."""
self.logger.info("Adding kv list to queue")
try:
from s3backgrounddelete.object_recovery_queue import ObjectRecoveryRabbitMq
mq_client = ObjectRecoveryRabbitMq(
self.config,
self.config.get_rabbitmq_username(),
self.config.get_rabbitmq_password(),
self.config.get_rabbitmq_host(),
self.config.get_rabbitmq_exchange(),
self.config.get_rabbitmq_queue_name(),
self.config.get_rabbitmq_mode(),
self.config.get_rabbitmq_durable(),
self.logger)
# Cleanup all entries and enqueue only 1000 entries
mq_client.purge_queue(self.config.get_rabbitmq_queue_name())
result, index_response = CORTXS3IndexApi(
self.config, logger=self.logger).list(
self.config.get_probable_delete_index_id(), self.config.get_max_keys(), marker)
if result:
self.logger.info("Index listing result :" +
str(index_response.get_index_content()))
probable_delete_json = index_response.get_index_content()
probable_delete_oid_list = probable_delete_json["Keys"]
is_truncated = probable_delete_json["IsTruncated"]
if (probable_delete_oid_list is not None):
for record in probable_delete_oid_list:
# Check if record is older than the pre-configured 'time to process' delay
leak_processing_delay = self.config.get_leak_processing_delay_in_mins()
try:
objLeakVal = json.loads(record["Value"])
except ValueError as error:
self.logger.error(
"Failed to parse JSON data for: " + str(record) + " due to: " + error)
continue
if (objLeakVal is None):
self.logger.error("No value associated with " + str(record) + ". Skipping entry")
continue
# Check if object leak entry is older than 15mins or a preconfigured duration
if (not ObjectRecoveryScheduler.isObjectLeakEntryOlderThan(objLeakVal, leak_processing_delay)):
self.logger.info("Object leak entry " + record["Key"] +
" is NOT older than " + str(leak_processing_delay) +
"mins. Skipping entry")
continue
self.logger.info(
"Object recovery queue sending data :" +
str(record))
ret, msg = mq_client.send_data(
record, self.config.get_rabbitmq_queue_name())
if not ret:
IEMutil("ERROR", IEMutil.RABBIT_MQ_CONN_FAILURE, IEMutil.RABBIT_MQ_CONN_FAILURE_STR)
self.logger.error(
"Object recovery queue send data "+ str(record) +
" failed :" + msg)
else:
self.logger.info(
"Object recovery queue send data successfully :" +
str(record))
else:
self.logger.info(
"Index listing result empty. Ignoring adding entry to object recovery queue")
pass
else:
self.logger.error("Failed to retrive Index listing:")
except BaseException:
self.logger.error(
"Object recovery queue send data exception:" + traceback.format_exc())
finally:
if mq_client:
self.logger.info("Closing the mqclient")
mq_client.close()