def tick(self):
try:
self._expire_backups_due()
except Exception, ex:
logger.exception("BackupExpirationManager Error")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(subject, message, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def tick(self):
try:
self._expire_backups_due()
except Exception, ex:
logger.exception("BackupExpirationManager Error")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(
subject,
message,
notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def run(self):
try:
app = self.flask_server
logger.info("%s: Starting HTTPServer (port=%s, protocol=%s)" % (self.name, self.port, self.protocol))
serve(app, host='0.0.0.0', port=self.port, url_scheme=self.protocol,
threads=self.num_workers, _server=self.custom_waitress_create_server)
except Exception, ex:
logger.exception("Api Server crashed")
sbj = "Api Server %s on %s crashed" % (self.name, get_local_host_name())
get_mbs().notifications.send_event_notification(sbj, sbj, priority=NotificationPriority.CRITICAL)
self.stop_api_server()
def _expire_backups_due(self):
logger.info("BackupExpirationManager: START EXPIRATION CHECK CYCLE")
# expire recurring backups
try:
self._expire_due_recurring_backups()
except Exception, ex:
logger.exception("BackupExpirationManager error during recurring backups expiration")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(subject, message, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def process_plan_retention(self, plan):
q = self._check_to_expire_query()
q["plan._id"] = plan.id
plan_backups = get_mbs().backup_collection.find(q)
self._process_plan(plan, plan_backups)
def _expire_due_onetime_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all onetime backups "
"due for expiration")
total_processed = 0
total_expired = 0
total_dont_expire = 0
q = self._check_to_expire_query()
q["plan._id"] = {
"$exists": False
}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
onetime_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
for onetime_backup in onetime_backups_iter:
if self.stop_requested:
break
total_processed += 1
if self.is_onetime_backup_not_expirable(onetime_backup):
self.mark_backup_never_expire(onetime_backup)
total_dont_expire += 1
elif self.is_onetime_backup_due_for_expiration(onetime_backup):
self.expire_backup(onetime_backup)
total_expired += 1
logger.info("BackupExpirationManager: Finished processing Onetime"
" Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def process_plan_retention(self, plan):
q = self._check_to_expire_query()
q["plan._id"] = plan.id
plan_backups = get_mbs().backup_collection.find(q)
self._process_plan(plan, plan_backups)
def delete_backup_targets(self, backup):
logger.info("Attempt to delete targets for backup '%s'" % backup.id)
self.validate_backup_target_delete(backup)
try:
if not self.test_mode:
self.robustified_delete_backup(backup)
return True
else:
logger.info("NOOP. Running in test mode. Not deleting "
"targets for backup '%s'" % backup.id)
except Exception, e:
msg = "Error while attempting to delete backup '%s': %s" % (backup.id, e)
logger.exception(msg)
get_mbs().notifications.send_notification("Backup Delete Error",
msg, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def _expire_due_onetime_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all onetime backups "
"due for expiration")
total_processed = 0
total_expired = 0
total_dont_expire = 0
q = self._check_to_expire_query()
q["plan._id"] = {"$exists": False}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
onetime_backups_iter = get_mbs().backup_collection.find_iter(
query=q, no_cursor_timeout=True)
for onetime_backup in onetime_backups_iter:
if self.stop_requested:
break
total_processed += 1
if self.is_onetime_backup_not_expirable(onetime_backup):
self.mark_backup_never_expire(onetime_backup)
total_dont_expire += 1
elif self.is_onetime_backup_due_for_expiration(onetime_backup):
self.expire_backup(onetime_backup)
total_expired += 1
logger.info("BackupExpirationManager: Finished processing Onetime"
" Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def _expire_backups_due(self):
logger.info("BackupExpirationManager: START EXPIRATION CHECK CYCLE")
# expire recurring backups
try:
self._expire_due_recurring_backups()
except Exception, ex:
logger.exception(
"BackupExpirationManager error during recurring backups expiration"
)
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(
subject,
message,
notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def start(self):
try:
app = self.flask_server
logger.info("%s: Starting HTTPServer (port=%s, protocol=%s, workers=%s)" %
(self.name, self.port, self.protocol, self.num_workers))
options = {
"bind": "0.0.0.0:%s" % self.port,
"workers": self.num_workers,
"worker_class": "gevent" if not self.debug_mode else "sync",
"proxy_protocol": self.protocol == "https"
}
MbsApiGunicornApplication(app, options).run()
except Exception, ex:
logger.exception("Api Server crashed")
sbj = "Api Server %s on %s crashed" % (self.name, get_local_host_name())
get_mbs().notifications.send_event_notification(sbj, sbj, priority=NotificationPriority.CRITICAL)
def delete_backup_targets(self, backup):
logger.info("Attempt to delete targets for backup '%s'" % backup.id)
self.validate_backup_target_delete(backup)
try:
if not self.test_mode:
self.robustified_delete_backup(backup)
return True
else:
logger.info("NOOP. Running in test mode. Not deleting "
"targets for backup '%s'" % backup.id)
except Exception, e:
msg = "Error while attempting to delete backup '%s': %s" % (
backup.id, e)
logger.exception(msg)
get_mbs().notifications.send_notification(
"Backup Delete Error",
msg,
notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
def _process_plan(self, plan, plan_backups):
total_dont_expire = 0
total_expired = 0
logger.info("==== Processing plan '%s' .... " % plan.id)
# Ensure we have the latest revision of the backup plan
plan = persistence.get_backup_plan(plan.id) or plan
try:
expirable_backups, non_expirable_backups = self.find_plan_expirable_backups(plan, plan_backups)
if non_expirable_backups:
self.mark_plan_backups_not_expirable(plan, non_expirable_backups)
total_dont_expire += len(non_expirable_backups)
total_expired += self.expire_plan_dues(plan, expirable_backups)
except Exception, e:
logger.exception("BackupExpirationManager Error while"
" processing plan '%s'" % plan.id)
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error while processing"
" plan '%s'\n\nStack Trace:\n%s" %
(plan.id, traceback.format_exc()))
get_mbs().notifications.send_error_notification(subject, message)
def start(self):
try:
app = self.flask_server
logger.info(
"%s: Starting HTTPServer (port=%s, protocol=%s, workers=%s)" %
(self.name, self.port, self.protocol, self.num_workers))
options = {
"bind": "0.0.0.0:%s" % self.port,
"workers": self.num_workers,
"worker_class": "gevent" if not self.debug_mode else "sync",
"proxy_protocol": self.protocol == "https"
}
MbsApiGunicornApplication(app, options).run()
except Exception, ex:
logger.exception("Api Server crashed")
sbj = "Api Server %s on %s crashed" % (self.name,
get_local_host_name())
get_mbs().notifications.send_event_notification(
sbj, sbj, priority=NotificationPriority.CRITICAL)
def _process_plan(self, plan, plan_backups):
total_dont_expire = 0
total_expired = 0
logger.info("==== Processing plan '%s' .... " % plan.id)
# Ensure we have the latest revision of the backup plan
plan = persistence.get_backup_plan(plan.id) or plan
try:
expirable_backups, non_expirable_backups = self.find_plan_expirable_backups(
plan, plan_backups)
if non_expirable_backups:
self.mark_plan_backups_not_expirable(plan,
non_expirable_backups)
total_dont_expire += len(non_expirable_backups)
total_expired += self.expire_plan_dues(plan, expirable_backups)
except Exception, e:
logger.exception("BackupExpirationManager Error while"
" processing plan '%s'" % plan.id)
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error while processing"
" plan '%s'\n\nStack Trace:\n%s" %
(plan.id, traceback.format_exc()))
get_mbs().notifications.send_error_notification(subject, message)
def _delete_backups_targets_due(self):
logger.info("BackupSweeper: Starting a sweep cycle...")
# clear stats
self._cycle_total_processed = 0
self._cycle_total_errored = 0
self._cycle_total_deleted = 0
# compute # of workers based on cpu count
self._worker_count = multiprocessing.cpu_count() * 2 + 1
self._sweep_workers = []
self._start_workers()
if self.test_mode:
logger.info("BackupSweeper: Running in TEST MODE. Nothing will"
" be really deleted")
logger.info("BackupSweeper: Finding all backups"
" due for deletion")
q = self._check_to_delete_query()
logger.info("BackupSweeper: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
backups_iterated = 0
# process all plan backups
for backup in backups_iter:
self._sweep_queue.put(backup)
backups_iterated += 1
# PERFORMANCE OPTIMIZATION
# process 10 * worker at max
# This is needed because making backup objects (from within the backups_iter) takes up a lot of CPU/Memory
# This is needed to give it a breath
if backups_iterated % (self._worker_count * 10) == 0:
self._wait_for_queue_to_be_empty()
self._finish_cycle()
logger.info("BackupSweeper: Finished sweep cycle. "
"Total Deleted=%s, Total Errored=%s, "
"Total Processed=%s" %
(self._cycle_total_deleted,
self._cycle_total_errored,
self._cycle_total_processed))
def _expire_due_recurring_backups(self):
total_processed = 0
total_expired = 0
total_dont_expire = 0
logger.info("BackupExpirationManager: Finding all recurring backups"
" due for expiration")
q = self._check_to_expire_query()
q["plan._id"] = {
"$exists": True
}
s = [("plan._id", -1)]
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(query=q, sort=s, no_cursor_timeout=True)
current_backup = next(backups_iter, None)
plan = current_backup.plan if current_backup else None
plan_backups = []
# process all plan backups
while current_backup and not self.stop_requested:
total_processed += 1
if current_backup.plan.id == plan.id:
plan_backups.append(current_backup)
current_backup = next(backups_iter, None)
# process the current plan
if not current_backup or current_backup.plan.id != plan.id:
plan_total_expired, plan_total_dont_expire = \
self._process_plan(plan, plan_backups)
total_expired += plan_total_expired
total_dont_expire = plan_total_dont_expire
plan = current_backup.plan if current_backup else None
plan_backups = []
logger.info("BackupExpirationManager: Finished processing Recurring "
"Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def _expire_due_recurring_backups(self):
total_processed = 0
total_expired = 0
total_dont_expire = 0
logger.info("BackupExpirationManager: Finding all recurring backups"
" due for expiration")
q = self._check_to_expire_query()
q["plan._id"] = {"$exists": True}
s = [("plan._id", -1)]
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(
query=q, sort=s, no_cursor_timeout=True)
current_backup = next(backups_iter, None)
plan = current_backup.plan if current_backup else None
plan_backups = []
# process all plan backups
while current_backup and not self.stop_requested:
total_processed += 1
if current_backup.plan.id == plan.id:
plan_backups.append(current_backup)
current_backup = next(backups_iter, None)
# process the current plan
if not current_backup or current_backup.plan.id != plan.id:
plan_total_expired, plan_total_dont_expire = \
self._process_plan(plan, plan_backups)
total_expired += plan_total_expired
total_dont_expire = plan_total_dont_expire
plan = current_backup.plan if current_backup else None
plan_backups = []
logger.info("BackupExpirationManager: Finished processing Recurring "
"Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def _delete_backups_targets_due(self):
logger.info("BackupSweeper: Starting a sweep cycle...")
# clear stats
self._cycle_total_processed = 0
self._cycle_total_errored = 0
self._cycle_total_deleted = 0
# compute # of workers based on cpu count
self._worker_count = multiprocessing.cpu_count() * 2 + 1
self._sweep_workers = []
self._start_workers()
if self.test_mode:
logger.info("BackupSweeper: Running in TEST MODE. Nothing will"
" be really deleted")
logger.info("BackupSweeper: Finding all backups" " due for deletion")
q = self._check_to_delete_query()
logger.info("BackupSweeper: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(
query=q, no_cursor_timeout=True)
backups_iterated = 0
# process all plan backups
for backup in backups_iter:
self._sweep_queue.put(backup)
backups_iterated += 1
# PERFORMANCE OPTIMIZATION
# process 10 * worker at max
# This is needed because making backup objects (from within the backups_iter) takes up a lot of CPU/Memory
# This is needed to give it a breath
if backups_iterated % (self._worker_count * 10) == 0:
self._wait_for_queue_to_be_empty()
self._finish_cycle()
logger.info("BackupSweeper: Finished sweep cycle. "
"Total Deleted=%s, Total Errored=%s, "
"Total Processed=%s" %
(self._cycle_total_deleted, self._cycle_total_errored,
self._cycle_total_processed))
def setUp(self):
self._temp_config_dir = self.__class__._createMBSDir()
self._temp_config_path = \
os.path.join(self._temp_config_dir, 'mbs.config')
self._temp_log_path = \
os.path.join(self._temp_config_dir, 'logs')
if self.__class__.CONFIG is not None:
open(self._temp_config_path, 'w').write(
json.dumps(self.__class__.CONFIG, indent=4, sort_keys=True))
self._orig_config_path = mbs_config.MBS_CONF_PATH
mbs_config.MBS_CONF_PATH = self._temp_config_path
self._orig_log_path = mbs_config.MBS_LOG_PATH
mbs_config.MBS_LOG_PATH = self._temp_log_path
self._mbs = mbs.get_mbs()
def _expire_due_canceled_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all canceled backups "
"due for expiration")
q = self._check_to_expire_query()
q["state"] = State.CANCELED
q["createdDate"] = {"$lt": self.expired_canceled_cutoff_date()}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
canceled_backups_iter = get_mbs().backup_collection.find_iter(
query=q, no_cursor_timeout=True)
for backup in canceled_backups_iter:
if self.stop_requested:
break
# for canceled backups, we always expire them immediately
self.expire_backup(backup)
logger.info("BackupExpirationManager: Finished processing canceled"
" Backups")
def _expire_due_canceled_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all canceled backups "
"due for expiration")
q = self._check_to_expire_query()
q["state"] = State.CANCELED
q["createdDate"] = {
"$lt": self.expired_canceled_cutoff_date()
}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
canceled_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
for backup in canceled_backups_iter:
if self.stop_requested:
break
# for canceled backups, we always expire them immediately
self.expire_backup(backup)
logger.info("BackupExpirationManager: Finished processing canceled"
" Backups")
def backup_system(self):
return get_mbs().backup_system
def status(self):
return {
"status": "running",
"versionInfo": get_mbs().get_version_info()
}
def send_api_error(end_point, exception):
subject = "BackupSystemAPI Error"
message = ("BackupSystemAPI Error on '%s'.\n\nStack Trace:\n%s" %
(end_point, traceback.format_exc()))
get_mbs().notifications.send_error_notification(subject, message)
message,
notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
try:
self._expire_due_onetime_backups()
except Exception, ex:
logger.exception(
"BackupExpirationManager error during onetime backups expiration"
)
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(
subject,
message,
notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
try:
self._expire_due_canceled_backups()
except Exception, ex:
logger.exception(
"BackupExpirationManager error during canceled backups expiration"
)
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(
subject,
message,
def backup_system(self):
return get_mbs().backup_system
def status(self):
return {
"status": "running",
"versionInfo": get_mbs().get_version_info()
}
logger.exception("BackupExpirationManager error during recurring backups expiration")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(subject, message, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
try:
self._expire_due_onetime_backups()
except Exception, ex:
logger.exception("BackupExpirationManager error during onetime backups expiration")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(subject, message, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
try:
self._expire_due_canceled_backups()
except Exception, ex:
logger.exception("BackupExpirationManager error during canceled backups expiration")
subject = "BackupExpirationManager Error"
message = ("BackupExpirationManager Error!.\n\nStack Trace:\n%s" %
traceback.format_exc())
get_mbs().notifications.send_notification(subject, message, notification_type=NotificationType.EVENT,
priority=NotificationPriority.CRITICAL)
logger.info("BackupExpirationManager: END EXPIRATION CHECK CYCLE")
###########################################################################
def _expire_due_recurring_backups(self):
