def stop_backup_system():
     logger.info("Command Server: Received a stop command")
     try:
         # stop the backup system
         backup_system.request_stop()
         return document_pretty_string({"ok": True})
     except Exception, e:
         msg = "Error while trying to stop backup system: %s" % e
         logger.error(msg)
         logger.error(traceback.format_exc())
         return document_pretty_string({"error": "can't stop"})
 def cancel_restore():
     restore_id = request.args.get('restoreId')
     logger.info("Command Server: Received a cancel-restore command")
     try:
         engine.cancel_restore(restore_id)
         return document_pretty_string({"ok": 1})
     except Exception, e:
         msg = ("Error while trying to cancel restore '%s': %s" %
                (restore_id, e))
         logger.error(msg)
         logger.error(traceback.format_exc())
         return document_pretty_string({"ok": 0, "error": msg})
 def stop_backup_system():
     logger.info("Command Server: Received a stop command")
     try:
         # stop the backup system
         backup_system.request_stop()
         return document_pretty_string({
             "ok": True
         })
     except Exception, e:
         msg = "Error while trying to stop backup system: %s" % e
         logger.error(msg)
         logger.error(traceback.format_exc())
         return document_pretty_string({"error": "can't stop"})
    def fsyncunlock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsyncunlock on %s" % self)

            if self.version_greater_than_3_2():
                result = self.admin_db.command(SON([("fsyncUnlock", 1)]))
            else:
                result = self.admin_db["$cmd.sys.unlock"].find_one()

            if result.get("ok"):
                logger.info("fsyncunlock ran successfully on %s" % self)
            else:
                msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
                       (self, document_pretty_string(result)))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsyncunlock on '%s'. %s" % (self, e)
                logger.error(msg)
                raise MongoLockError(msg, cause=e)
            else:
                raise
    def fsynclock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsynclock on %s" % self)

            if self.is_server_locked():
                raise ServerAlreadyLockedError("Cannot run fsynclock on server '%s' "
                                               "because its already locked!" % self)
            result = self.admin_db.command(SON([("fsync", 1),("lock", True)]))


            if result.get("ok"):
                logger.info("fsynclock ran successfully on %s" % self)
            else:
                msg = ("fsynclock was not successful on '%s'. Result: %s" %
                       document_pretty_string(result))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsynclock on '%s'. %s" % (self, e)
                logger.error(msg)
                raise MongoLockError(msg, cause=e)
            else:
                raise
    def run(self):
        self.info("Starting up... ")
        self.info("PID is %s" % os.getpid())
        self.info("TEMP DIR is '%s'" % self.temp_dir)
        if self.tags:
            self.info("Tags are: %s" % document_pretty_string(self.tags))
        else:
            self.info("No tags configured")

        ensure_dir(self._temp_dir)
        self._update_pid_file()
        # Start the command server
        self._start_command_server()

        # start the backup processor
        self._backup_processor.start()

        # start the restore processor
        self._restore_processor.start()

        # start the backup processor
        self._backup_processor.join()

        # start the restore processor
        self._restore_processor.join()

        self.info("Engine completed")
        self._pre_shutdown()
    def _expire_due_onetime_backups(self):
        # process onetime backups
        logger.info("BackupExpirationManager: Finding all onetime backups "
                    "due for expiration")

        total_processed = 0
        total_expired = 0
        total_dont_expire = 0
        q = _check_to_expire_query()

        q["plan._id"] = {
            "$exists": False
        }

        logger.info("BackupExpirationManager: Executing query :\n%s" %
                    document_pretty_string(q))
        onetime_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)

        for onetime_backup in onetime_backups_iter:
            if self.stop_requested:
                break

            total_processed += 1
            if self.is_onetime_backup_not_expirable(onetime_backup):
                mark_backup_never_expire(onetime_backup)
                total_dont_expire += 1
            elif self.is_onetime_backup_due_for_expiration(onetime_backup):
                self.expire_backup(onetime_backup)
                total_expired += 1

        logger.info("BackupExpirationManager: Finished processing Onetime"
                    " Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
                    "Total Processed=%s" %
                    (total_expired, total_dont_expire, total_processed))
    def fsyncunlock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsyncunlock on %s" % self)

            if self.version_greater_than_3_2():
                result = self.admin_db.command(SON([("fsyncUnlock", 1)]))
            else:
                result = self.admin_db["$cmd.sys.unlock"].find_one()

            if result.get("ok"):
                logger.info("fsyncunlock ran successfully on %s" % self)
            else:
                msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
                       (self, document_pretty_string(result)))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsyncunlock on '%s'. %s" % (self,
                                                                         e)
                logger.error(msg)
                raise MongoLockError(msg, cause=e)
            else:
                raise
    def run(self):
        self.info("Starting up... ")
        self.info("PID is %s" % os.getpid())
        self.info("TEMP DIR is '%s'" % self.temp_dir)
        if self.tags:
            self.info("Tags are: %s" % document_pretty_string(self.tags))
        else:
            self.info("No tags configured")

        ensure_dir(self._temp_dir)
        self._update_pid_file()
        # Start the command server
        self._start_command_server()

        # start the backup processor
        self._backup_processor.start()

        # start the restore processor
        self._restore_processor.start()

        # start the backup processor
        self._backup_processor.join()

        # start the restore processor
        self._restore_processor.join()

        self.info("Engine completed")
        self._pre_shutdown()
 def stop_backup_system():
     logger.info("Command Server: Received a stop command")
     try:
         backup_system._do_stop()
         return document_pretty_string({"ok": True})
     except Exception, e:
         return "Error while trying to stop backup system: %s" % e
    def fsynclock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsynclock on %s" % self)

            if self.is_server_locked():
                raise ServerAlreadyLockedError(
                    "Cannot run fsynclock on server '%s' "
                    "because its already locked!" % self)
            result = self.admin_db.command(SON([("fsync", 1), ("lock", True)]))

            if result.get("ok"):
                logger.info("fsynclock ran successfully on %s" % self)
            else:
                msg = ("fsynclock was not successful on '%s'. Result: %s" %
                       document_pretty_string(result))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsynclock on '%s'. %s" % (self, e)
                logger.error(msg)
                raise MongoLockError(msg, cause=e)
            else:
                raise
 def delete_backup(backup_id):
     logger.info("Command Server: Received a delete-backup command")
     try:
         result = backup_system.delete_backup(backup_id)
         return document_pretty_string(result)
     except Exception, e:
         return ("Error while trying to delete backup %s: %s" %
                 (backup_id, e))
Example #13
0
 def cancel_backup():
     backup_id = request.args.get('backupId')
     logger.info("Command Server: Received a cancel-backup command")
     try:
         engine.cancel_backup(backup_id)
         return document_pretty_string({
             "ok": 1
         })
     except Exception, e:
         msg = ("Error while trying to cancel backup '%s': %s" %
               (backup_id, e))
         logger.error(msg)
         logger.error(traceback.format_exc())
         return document_pretty_string({
             "ok": 0,
             "error": msg
         })
 def status():
     logger.info("Command Server: Received a status command")
     try:
         return document_pretty_string(backup_system.do_get_status())
     except Exception, e:
         msg = "Error while trying to get backup system status: %s" % e
         logger.error(msg)
         logger.error(traceback.format_exc())
         return {"status": "UNKNOWN", "error": msg}
Example #15
0
def new_server(server_doc):
    _type = server_doc.get("_type")

    if _type is None or _type in SERVER_TYPE_MAP:
        clazz = resolve_class(SERVER_TYPE_MAP.get(_type,
                                                  MONGOD_SERVER_CLASS_NAME))
    else:
        raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(server_doc)))

    return clazz(server_doc)
Example #16
0
def new_server(server_doc):
    _type = server_doc.get("_type")

    if _type is None or _type in SERVER_TYPE_MAP:
        clazz = resolve_class(
            SERVER_TYPE_MAP.get(_type, MONGOD_SERVER_CLASS_NAME))
    else:
        raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(server_doc)))

    return clazz(server_doc)
Example #17
0
def new_cluster(cluster_doc):
    _type = cluster_doc.get("_type")

    if _type is None or _type == "ReplicaSetCluster":
        clazz = replicaset_cluster_type()
    elif _type == "ShardedCluster":
        clazz = sharded_cluster_type()
    else:
        raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(cluster_doc)))
    return clazz(cluster_doc)
Example #18
0
def new_cluster(cluster_doc):
    _type = cluster_doc.get("_type")

    if _type is None or _type == "ReplicaSetCluster":
        clazz = replicaset_cluster_type()
    elif _type == "ShardSetCluster":
        clazz = shardset_cluster_type()
    else:
        raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(cluster_doc)))
    return clazz(cluster_doc)
    def info(self):
        i = {"router": self.router.info()}

        if self.config_server:
            i["configServer"] = self.config_server.info()

        if self.selected_shard_secondaries:
            shard_infos = map(lambda s: s.info(),
                              self.selected_shard_secondaries)
            i["selectedShardSecondaries"] = shard_infos

        return document_pretty_string(i)
 def status():
     logger.info("Command Server: Received a status command")
     try:
         return document_pretty_string(backup_system.do_get_status())
     except Exception, e:
         msg = "Error while trying to get backup system status: %s" % e
         logger.error(msg)
         logger.error(traceback.format_exc())
         return {
             "status": "UNKNOWN",
             "error": msg
         }
Example #21
0
    def info(self):
        i = {
            "router": self.router.info(),
            "configServer": self.config_server.info()
        }

        if self.selected_shard_secondaries:
            shard_infos = map(lambda s: s.info(),
                              self.selected_shard_secondaries)
            i["selectedShardSecondaries"] = shard_infos

        return document_pretty_string(i)
Example #22
0
def new_server(server_doc):
    _type = server_doc.get("_type")

    if _type is None or _type == "mongod":
        server_type = "mongoctl.objects.mongod.MongodServer"
    elif _type == "mongos":
        server_type = "mongoctl.objects.mongos.MongosServer"
    else:
        raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(server_doc)))

    clazz = resolve_class(server_type)
    return clazz(server_doc)
Example #23
0
def new_server(server_doc):
    _type = server_doc.get("_type")

    if _type is None or _type == "mongod":
        server_type = "mongoctl.objects.mongod.MongodServer"
    elif _type == "mongos":
        server_type = "mongoctl.objects.mongos.MongosServer"
    else:
        raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
                                (_type, document_pretty_string(server_doc)))

    clazz = resolve_class(server_type)
    return clazz(server_doc)
    def _delete_backups_targets_due(self):

        logger.info("BackupSweeper: Starting a sweep cycle...")

        # clear stats
        self._cycle_total_processed = 0
        self._cycle_total_errored = 0
        self._cycle_total_deleted = 0
        self._sweep_workers = []

        self._start_workers()

        if self.test_mode:
            logger.info("BackupSweeper: Running in TEST MODE. Nothing will"
                        " be really deleted")

        logger.info("BackupSweeper: Finding all backups"
                    " due for deletion")
        q = self._check_to_delete_query()

        logger.info("BackupSweeper: Executing query :\n%s" %
                    document_pretty_string(q))

        backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)

        backups_iterated = 0
        # process all plan backups
        for backup in backups_iter:
            if self.stop_requested:
                break

            self._sweep_queue.put(backup)
            backups_iterated += 1
            # PERFORMANCE OPTIMIZATION
            # process 10 at max at a time
            # This is needed because making backup objects (from within the backups_iter) takes up a lot of CPU/Memory
            # This is needed to give it a breath
            if backups_iterated % 10 == 0:
                self._wait_for_queue_to_be_empty()

        self._finish_cycle()


        logger.info("BackupSweeper: Finished sweep cycle. "
                    "Total Deleted=%s, Total Errored=%s, "
                    "Total Processed=%s" %
                    (self._cycle_total_deleted,
                    self._cycle_total_errored,
                    self._cycle_total_processed))
    def _expire_due_recurring_backups(self):

        total_processed = 0
        total_expired = 0
        total_dont_expire = 0

        logger.info("BackupExpirationManager: Finding all recurring backups"
                    " due for expiration")
        q = _check_to_expire_query()

        q["plan._id"] = {
            "$exists": True
        }

        s = [("plan._id", -1)]

        logger.info("BackupExpirationManager: Executing query :\n%s" %
                    document_pretty_string(q))

        backups_iter = get_mbs().backup_collection.find_iter(query=q, sort=s, no_cursor_timeout=True)

        current_backup = next(backups_iter, None)

        plan = current_backup.plan if current_backup else None
        plan_backups = []

        # process all plan backups
        while current_backup and not self.stop_requested:
            total_processed += 1

            if current_backup.plan.id == plan.id:
                plan_backups.append(current_backup)

            current_backup = next(backups_iter, None)
            # process the current plan
            if not current_backup or current_backup.plan.id != plan.id:
                plan_total_expired, plan_total_dont_expire = \
                    self._process_plan(plan, plan_backups)
                total_expired += plan_total_expired
                total_dont_expire = plan_total_dont_expire

                plan = current_backup.plan if current_backup else None
                plan_backups = []

        logger.info("BackupExpirationManager: Finished processing Recurring "
                    "Backups.\nTotal Expired=%s, Total Don't Expire=%s, "
                    "Total Processed=%s" %
                    (total_expired, total_dont_expire, total_processed))
    def fsyncunlock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsyncunlock on %s" % self)

            result = self._admin_db["$cmd.sys.unlock"].find_one()

            if result["ok"]:
                logger.info("fsyncunlock ran successfully on %s" % self)
            else:
                msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
                       document_pretty_string(result))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsyncunlock on '%s'."
                raise MongoLockError(msg=msg, cause=e)
            else:
                raise
Example #27
0
    def fsyncunlock(self):
        """
            Runs fsynclock command on the server
        """

        try:
            logger.info("Attempting to run fsyncunlock on %s" % self)

            result = self._admin_db["$cmd.sys.unlock"].find_one()

            if result["ok"]:
                logger.info("fsyncunlock ran successfully on %s" % self)
            else:
                msg = ("fsyncunlock was not successful on '%s'. Result: %s" %
                       document_pretty_string(result))
                raise MongoLockError(msg)
        except Exception, e:
            if not isinstance(e, MongoLockError):
                msg = "Error while executing fsyncunlock on '%s'."
                raise MongoLockError(msg=msg, cause=e)
            else:
                raise
Example #28
0
    def run(self):
        self.info("Starting up... ")
        self.info("PID is %s" % os.getpid())
        self.info("PATH is '%s'" % os.environ['PATH'])
        if self.tags:
            self.info("Tags are: %s" % document_pretty_string(self.tags))
        else:
            self.info("No tags configured")

        # ensure task log dirs
        ensure_dir(task_log_dir("backup"))
        ensure_dir(task_log_dir("restore"))

        self._update_pid_file()
        # Start the command server
        self._start_command_server()

        self.start_task_processors()

        self.wait_task_processors()

        self.info("Engine completed")
        self._pre_shutdown()
    def _expire_due_canceled_backups(self):
        # process onetime backups
        logger.info("BackupExpirationManager: Finding all canceled backups "
                    "due for expiration")

        q = _check_to_expire_query()

        q["state"] = State.CANCELED
        q["createdDate"] = {
            "$lt": self.expired_canceled_cutoff_date()
        }

        logger.info("BackupExpirationManager: Executing query :\n%s" %
                    document_pretty_string(q))
        canceled_backups_iter = get_mbs().backup_collection.find_iter(query=q, no_cursor_timeout=True)

        for backup in canceled_backups_iter:
            if self.stop_requested:
                break
            # for canceled backups, we always expire them immediately
            self.expire_backup(backup)

        logger.info("BackupExpirationManager: Finished processing canceled"
                    " Backups")
Example #30
0
 def status():
     logger.info("Command Server: Received a status command")
     try:
         return document_pretty_string(engine._do_get_status())
     except Exception, e:
         return "Error while trying to get engine status: %s" % e
Example #31
0
 def __str__(self):
     return document_pretty_string(self.to_document(display_only=True))
Example #32
0
def log_db_command(cmd):
    log_info( "Executing db command %s" % utils.document_pretty_string(cmd))
Example #33
0
 def __str__(self):
     return utils.document_pretty_string(self.to_document(display_only=True))
Example #34
0
 def status():
     logger.info("Command Server: Received a status command")
     try:
         return document_pretty_string(engine._do_get_status())
     except Exception, e:
         return "Error while trying to get engine status: %s" % e
Example #35
0
def log_db_command(cmd):
    log_info("Executing db command %s" % utils.document_pretty_string(cmd))