def stop_backup_system():
logger.info("Command Server: Received a stop command")
try:
# stop the backup system
backup_system.request_stop()
return document_pretty_string({"ok": True})
except Exception, e:
msg = "Error while trying to stop backup system: %s" % e
logger.error(msg)
logger.error(traceback.format_exc())
return document_pretty_string({"error": "can't stop"})
def cancel_restore():
restore_id = request.args.get('restoreId')
logger.info("Command Server: Received a cancel-restore command")
try:
engine.cancel_restore(restore_id)
return document_pretty_string({"ok": 1})
except Exception, e:
msg = ("Error while trying to cancel restore '%s': %s" %
(restore_id, e))
logger.error(msg)
logger.error(traceback.format_exc())
return document_pretty_string({"ok": 0, "error": msg})
def stop_backup_system():
logger.info("Command Server: Received a stop command")
try:
# stop the backup system
backup_system.request_stop()
return document_pretty_string({
"ok": True
})
except Exception, e:
msg = "Error while trying to stop backup system: %s" % e
logger.error(msg)
logger.error(traceback.format_exc())
return document_pretty_string({"error": "can't stop"})
def fsyncunlock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsyncunlock on %s" % self)
if self.version_greater_than_3_2():
result = self.admin_db.command(SON([("fsyncUnlock", 1)]))
else:
result = self.admin_db["$cmd.sys.unlock"].find_one()
if result.get("ok"):
logger.info("fsyncunlock ran successfully on %s" % self)
else:
msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
(self, document_pretty_string(result)))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsyncunlock on '%s'. %s" % (self, e)
logger.error(msg)
raise MongoLockError(msg, cause=e)
else:
raise
def fsynclock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsynclock on %s" % self)
if self.is_server_locked():
raise ServerAlreadyLockedError("Cannot run fsynclock on server '%s' "
"because its already locked!" % self)
result = self.admin_db.command(SON([("fsync", 1),("lock", True)]))
if result.get("ok"):
logger.info("fsynclock ran successfully on %s" % self)
else:
msg = ("fsynclock was not successful on '%s'. Result: %s" %
document_pretty_string(result))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsynclock on '%s'. %s" % (self, e)
logger.error(msg)
raise MongoLockError(msg, cause=e)
else:
raise
def run(self):
self.info("Starting up... ")
self.info("PID is %s" % os.getpid())
self.info("TEMP DIR is '%s'" % self.temp_dir)
if self.tags:
self.info("Tags are: %s" % document_pretty_string(self.tags))
else:
self.info("No tags configured")
ensure_dir(self._temp_dir)
self._update_pid_file()
# Start the command server
self._start_command_server()
# start the backup processor
self._backup_processor.start()
# start the restore processor
self._restore_processor.start()
# start the backup processor
self._backup_processor.join()
# start the restore processor
self._restore_processor.join()
self.info("Engine completed")
self._pre_shutdown()
def _expire_due_onetime_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all onetime backups "
"due for expiration")
total_processed = 0
total_expired = 0
total_dont_expire = 0
q = _check_to_expire_query()
q["plan._id"] = {
"$exists": False
}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
onetime_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
for onetime_backup in onetime_backups_iter:
if self.stop_requested:
break
total_processed += 1
if self.is_onetime_backup_not_expirable(onetime_backup):
mark_backup_never_expire(onetime_backup)
total_dont_expire += 1
elif self.is_onetime_backup_due_for_expiration(onetime_backup):
self.expire_backup(onetime_backup)
total_expired += 1
logger.info("BackupExpirationManager: Finished processing Onetime"
" Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def fsyncunlock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsyncunlock on %s" % self)
if self.version_greater_than_3_2():
result = self.admin_db.command(SON([("fsyncUnlock", 1)]))
else:
result = self.admin_db["$cmd.sys.unlock"].find_one()
if result.get("ok"):
logger.info("fsyncunlock ran successfully on %s" % self)
else:
msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
(self, document_pretty_string(result)))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsyncunlock on '%s'. %s" % (self,
e)
logger.error(msg)
raise MongoLockError(msg, cause=e)
else:
raise
def run(self):
self.info("Starting up... ")
self.info("PID is %s" % os.getpid())
self.info("TEMP DIR is '%s'" % self.temp_dir)
if self.tags:
self.info("Tags are: %s" % document_pretty_string(self.tags))
else:
self.info("No tags configured")
ensure_dir(self._temp_dir)
self._update_pid_file()
# Start the command server
self._start_command_server()
# start the backup processor
self._backup_processor.start()
# start the restore processor
self._restore_processor.start()
# start the backup processor
self._backup_processor.join()
# start the restore processor
self._restore_processor.join()
self.info("Engine completed")
self._pre_shutdown()
def stop_backup_system():
logger.info("Command Server: Received a stop command")
try:
backup_system._do_stop()
return document_pretty_string({"ok": True})
except Exception, e:
return "Error while trying to stop backup system: %s" % e
def fsynclock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsynclock on %s" % self)
if self.is_server_locked():
raise ServerAlreadyLockedError(
"Cannot run fsynclock on server '%s' "
"because its already locked!" % self)
result = self.admin_db.command(SON([("fsync", 1), ("lock", True)]))
if result.get("ok"):
logger.info("fsynclock ran successfully on %s" % self)
else:
msg = ("fsynclock was not successful on '%s'. Result: %s" %
document_pretty_string(result))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsynclock on '%s'. %s" % (self, e)
logger.error(msg)
raise MongoLockError(msg, cause=e)
else:
raise
def delete_backup(backup_id):
logger.info("Command Server: Received a delete-backup command")
try:
result = backup_system.delete_backup(backup_id)
return document_pretty_string(result)
except Exception, e:
return ("Error while trying to delete backup %s: %s" %
(backup_id, e))
def cancel_backup():
backup_id = request.args.get('backupId')
logger.info("Command Server: Received a cancel-backup command")
try:
engine.cancel_backup(backup_id)
return document_pretty_string({
"ok": 1
})
except Exception, e:
msg = ("Error while trying to cancel backup '%s': %s" %
(backup_id, e))
logger.error(msg)
logger.error(traceback.format_exc())
return document_pretty_string({
"ok": 0,
"error": msg
})
def status():
logger.info("Command Server: Received a status command")
try:
return document_pretty_string(backup_system.do_get_status())
except Exception, e:
msg = "Error while trying to get backup system status: %s" % e
logger.error(msg)
logger.error(traceback.format_exc())
return {"status": "UNKNOWN", "error": msg}
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type in SERVER_TYPE_MAP:
clazz = resolve_class(SERVER_TYPE_MAP.get(_type,
MONGOD_SERVER_CLASS_NAME))
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
return clazz(server_doc)
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type in SERVER_TYPE_MAP:
clazz = resolve_class(
SERVER_TYPE_MAP.get(_type, MONGOD_SERVER_CLASS_NAME))
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
return clazz(server_doc)
def new_cluster(cluster_doc):
_type = cluster_doc.get("_type")
if _type is None or _type == "ReplicaSetCluster":
clazz = replicaset_cluster_type()
elif _type == "ShardedCluster":
clazz = sharded_cluster_type()
else:
raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" %
(_type, document_pretty_string(cluster_doc)))
return clazz(cluster_doc)
def new_cluster(cluster_doc):
_type = cluster_doc.get("_type")
if _type is None or _type == "ReplicaSetCluster":
clazz = replicaset_cluster_type()
elif _type == "ShardSetCluster":
clazz = shardset_cluster_type()
else:
raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" %
(_type, document_pretty_string(cluster_doc)))
return clazz(cluster_doc)
def info(self):
i = {"router": self.router.info()}
if self.config_server:
i["configServer"] = self.config_server.info()
if self.selected_shard_secondaries:
shard_infos = map(lambda s: s.info(),
self.selected_shard_secondaries)
i["selectedShardSecondaries"] = shard_infos
return document_pretty_string(i)
def status():
logger.info("Command Server: Received a status command")
try:
return document_pretty_string(backup_system.do_get_status())
except Exception, e:
msg = "Error while trying to get backup system status: %s" % e
logger.error(msg)
logger.error(traceback.format_exc())
return {
"status": "UNKNOWN",
"error": msg
}
def info(self):
i = {
"router": self.router.info(),
"configServer": self.config_server.info()
}
if self.selected_shard_secondaries:
shard_infos = map(lambda s: s.info(),
self.selected_shard_secondaries)
i["selectedShardSecondaries"] = shard_infos
return document_pretty_string(i)
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type == "mongod":
server_type = "mongoctl.objects.mongod.MongodServer"
elif _type == "mongos":
server_type = "mongoctl.objects.mongos.MongosServer"
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
clazz = resolve_class(server_type)
return clazz(server_doc)
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type == "mongod":
server_type = "mongoctl.objects.mongod.MongodServer"
elif _type == "mongos":
server_type = "mongoctl.objects.mongos.MongosServer"
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
clazz = resolve_class(server_type)
return clazz(server_doc)
def _delete_backups_targets_due(self):
logger.info("BackupSweeper: Starting a sweep cycle...")
# clear stats
self._cycle_total_processed = 0
self._cycle_total_errored = 0
self._cycle_total_deleted = 0
self._sweep_workers = []
self._start_workers()
if self.test_mode:
logger.info("BackupSweeper: Running in TEST MODE. Nothing will"
" be really deleted")
logger.info("BackupSweeper: Finding all backups"
" due for deletion")
q = self._check_to_delete_query()
logger.info("BackupSweeper: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
backups_iterated = 0
# process all plan backups
for backup in backups_iter:
if self.stop_requested:
break
self._sweep_queue.put(backup)
backups_iterated += 1
# PERFORMANCE OPTIMIZATION
# process 10 at max at a time
# This is needed because making backup objects (from within the backups_iter) takes up a lot of CPU/Memory
# This is needed to give it a breath
if backups_iterated % 10 == 0:
self._wait_for_queue_to_be_empty()
self._finish_cycle()
logger.info("BackupSweeper: Finished sweep cycle. "
"Total Deleted=%s, Total Errored=%s, "
"Total Processed=%s" %
(self._cycle_total_deleted,
self._cycle_total_errored,
self._cycle_total_processed))
def _expire_due_recurring_backups(self):
total_processed = 0
total_expired = 0
total_dont_expire = 0
logger.info("BackupExpirationManager: Finding all recurring backups"
" due for expiration")
q = _check_to_expire_query()
q["plan._id"] = {
"$exists": True
}
s = [("plan._id", -1)]
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
backups_iter = get_mbs().backup_collection.find_iter(query=q, sort=s, no_cursor_timeout=True)
current_backup = next(backups_iter, None)
plan = current_backup.plan if current_backup else None
plan_backups = []
# process all plan backups
while current_backup and not self.stop_requested:
total_processed += 1
if current_backup.plan.id == plan.id:
plan_backups.append(current_backup)
current_backup = next(backups_iter, None)
# process the current plan
if not current_backup or current_backup.plan.id != plan.id:
plan_total_expired, plan_total_dont_expire = \
self._process_plan(plan, plan_backups)
total_expired += plan_total_expired
total_dont_expire = plan_total_dont_expire
plan = current_backup.plan if current_backup else None
plan_backups = []
logger.info("BackupExpirationManager: Finished processing Recurring "
"Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
"Total Processed=%s" %
(total_expired, total_dont_expire, total_processed))
def fsyncunlock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsyncunlock on %s" % self)
result = self._admin_db["$cmd.sys.unlock"].find_one()
if result["ok"]:
logger.info("fsyncunlock ran successfully on %s" % self)
else:
msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
document_pretty_string(result))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsyncunlock on '%s'."
raise MongoLockError(msg=msg, cause=e)
else:
raise
def fsyncunlock(self):
"""
Runs fsynclock command on the server
"""
try:
logger.info("Attempting to run fsyncunlock on %s" % self)
result = self._admin_db["$cmd.sys.unlock"].find_one()
if result["ok"]:
logger.info("fsyncunlock ran successfully on %s" % self)
else:
msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
document_pretty_string(result))
raise MongoLockError(msg)
except Exception, e:
if not isinstance(e, MongoLockError):
msg = "Error while executing fsyncunlock on '%s'."
raise MongoLockError(msg=msg, cause=e)
else:
raise
def run(self):
self.info("Starting up... ")
self.info("PID is %s" % os.getpid())
self.info("PATH is '%s'" % os.environ['PATH'])
if self.tags:
self.info("Tags are: %s" % document_pretty_string(self.tags))
else:
self.info("No tags configured")
# ensure task log dirs
ensure_dir(task_log_dir("backup"))
ensure_dir(task_log_dir("restore"))
self._update_pid_file()
# Start the command server
self._start_command_server()
self.start_task_processors()
self.wait_task_processors()
self.info("Engine completed")
self._pre_shutdown()
def _expire_due_canceled_backups(self):
# process onetime backups
logger.info("BackupExpirationManager: Finding all canceled backups "
"due for expiration")
q = _check_to_expire_query()
q["state"] = State.CANCELED
q["createdDate"] = {
"$lt": self.expired_canceled_cutoff_date()
}
logger.info("BackupExpirationManager: Executing query :\n%s" %
document_pretty_string(q))
canceled_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)
for backup in canceled_backups_iter:
if self.stop_requested:
break
# for canceled backups, we always expire them immediately
self.expire_backup(backup)
logger.info("BackupExpirationManager: Finished processing canceled"
" Backups")
def status():
logger.info("Command Server: Received a status command")
try:
return document_pretty_string(engine._do_get_status())
except Exception, e:
return "Error while trying to get engine status: %s" % e
def __str__(self):
return document_pretty_string(self.to_document(display_only=True))
def log_db_command(cmd):
log_info( "Executing db command %s" % utils.document_pretty_string(cmd))
def __str__(self):
return utils.document_pretty_string(self.to_document(display_only=True))
def status():
logger.info("Command Server: Received a status command")
try:
return document_pretty_string(engine._do_get_status())
except Exception, e:
return "Error while trying to get engine status: %s" % e
def log_db_command(cmd):
log_info("Executing db command %s" % utils.document_pretty_string(cmd))