Example #1
0
class MysqlLVMBackup(object):
    """A Holland Backup plugin suitable for performing LVM snapshots of a 
    filesystem underlying a live MySQL instance.

    This plugin produces tar archives of a MySQL data directory.
    """
    CONFIGSPEC = CONFIGSPEC

    def __init__(self, name, config, target_directory, dry_run=False):
        self.config = config
        self.config.validate_config(self.configspec())
        LOG.debug("Validated config: %r", self.config)
        self.name = name
        self.target_directory = target_directory
        self.dry_run = dry_run
        self.client = connect_simple(self.config['mysql:client'])

    def estimate_backup_size(self):
        """Estimate the backup size this plugin will produce

        This is currently the total directory size of the MySQL datadir
        """
        try:
            self.client.connect()
            datadir = self.client.show_variable('datadir')
            self.client.disconnect()
        except MySQLError, exc:
            raise BackupError("[%d] %s" % exc.args)
        return directory_size(datadir)
Example #2
0
 def estimate_backup_size(self):
     """Estimate the size of the backup this plugin will produce"""
     try:
         mysql_config = build_mysql_config(self.config['mysql:client'])
         client = connect(mysql_config['client'])
         datadir = client.show_variable('datadir')
         return directory_size(datadir)
     except MySQLError, exc:
         raise BackupError("Failed to lookup the MySQL datadir when "
                           "estimating backup size: [%d] %s" % exc.args)
Example #3
0
 def estimate_backup_size(self):
     """Estimate the size of the backup this plugin will produce"""
     try:
         mysql_config = build_mysql_config(self.config['mysql:client'])
         client = connect(mysql_config['client'])
         datadir = client.show_variable('datadir')
         return directory_size(datadir)
     except MySQLError, exc:
         raise BackupError("Failed to lookup the MySQL datadir when "
                           "estimating backup size: [%d] %s" % exc.args)
Example #4
0
    def free_required_space(self, name, required_bytes, dry_run=False):
        """Attempt to free at least ``required_bytes`` of old backups from a backupset

        :param name: name of the backupset to free space from
        :param required_bytes: integer number of bytes required for the backupset path
        :param dry_run: if true, this will only generate log messages but won't actually free space
        :returns: bool; True if freed or False otherwise
        """
        LOG.info(
            "Insufficient disk space for adjusted estimated backup size: %s",
            format_bytes(required_bytes),
        )
        LOG.info(
            "purge-on-demand is enabled. Discovering old backups to purge.")
        available_bytes = disk_free(os.path.join(self.spool.path, name))
        to_purge = {}
        for backup in self.spool.list_backups(name):
            backup_size = directory_size(backup.path)
            LOG.info("Found backup '%s': %s", backup.path,
                     format_bytes(backup_size))
            available_bytes += backup_size
            to_purge[backup] = backup_size
            if available_bytes > required_bytes:
                break
        else:
            LOG.info("Purging would only recover an additional %s",
                     format_bytes(sum(to_purge.values())))
            LOG.info(
                "Only %s total would be available, but the current "
                "backup requires %s",
                format_bytes(available_bytes),
                format_bytes(required_bytes),
            )
            return False

        purge_bytes = sum(to_purge.values())
        LOG.info(
            "Found %d backups to purge which will recover %s",
            len(to_purge),
            format_bytes(purge_bytes),
        )

        for backup in to_purge:
            if dry_run:
                LOG.info("Would purge: %s", backup.path)
            else:
                LOG.info("Purging: %s", backup.path)
                backup.purge()
        LOG.info(
            "%s now has %s of available space",
            os.path.join(self.spool.path, name),
            format_bytes(disk_free(os.path.join(self.spool.path, name))),
        )
        return True
Example #5
0
    def estimate_backup_size(self):
        """Estimate the backup size this plugin will produce

        This is currently the total directory size of the MySQL datadir
        """
        try:
            self.client.connect()
            datadir = self.client.show_variable("datadir")
            self.client.disconnect()
        except MySQLError as exc:
            raise BackupError("[%d] %s" % exc.args)
        return directory_size(datadir)
Example #6
0
 def estimate_backup_size(self):
     """Return estimated backup size"""
     mysql_config = build_mysql_config(self.config["mysql:client"])
     client = connect(mysql_config["client"])
     try:
         datadir = client.show_variable("datadir")
         return directory_size(datadir)
     except OSError as exc:
         raise BackupError("Failed to calculate directory size: [%d] %s" %
                           (exc.errno, exc.strerror))
     finally:
         client.close()
Example #7
0
    def estimate_backup_size(self):
        """Estimate the backup size this plugin will produce

        This is currently the total directory size of the MySQL datadir
        """
        try:
            self.client.connect()
            datadir = self.client.show_variable('datadir')
            self.client.disconnect()
        except MySQLError as exc:
            raise BackupError("[%d] %s" % exc.args)
        return directory_size(datadir)
Example #8
0
    def free_required_space(self, name, required_bytes, dry_run=False):
        """Attempt to free at least ``required_bytes`` of old backups from a backupset

        :param name: name of the backupset to free space from
        :param required_bytes: integer number of bytes required for the backupset path
        :param dry_run: if true, this will only generate log messages but won't actually free space
        :returns: bool; True if freed or False otherwise
        """
        LOG.info(
            "Insufficient disk space for adjusted estimated backup size: %s",
            format_bytes(required_bytes),
        )
        LOG.info("purge-on-demand is enabled. Discovering old backups to purge.")
        available_bytes = disk_free(os.path.join(self.spool.path, name))
        to_purge = {}
        for backup in self.spool.list_backups(name):
            backup_size = directory_size(backup.path)
            LOG.info("Found backup '%s': %s", backup.path, format_bytes(backup_size))
            available_bytes += backup_size
            to_purge[backup] = backup_size
            if available_bytes > required_bytes:
                break
        else:
            LOG.info(
                "Purging would only recover an additional %s", format_bytes(sum(to_purge.values()))
            )
            LOG.info(
                "Only %s total would be available, but the current " "backup requires %s",
                format_bytes(available_bytes),
                format_bytes(required_bytes),
            )
            return False

        purge_bytes = sum(to_purge.values())
        LOG.info(
            "Found %d backups to purge which will recover %s",
            len(to_purge),
            format_bytes(purge_bytes),
        )

        for backup in to_purge:
            if dry_run:
                LOG.info("Would purge: %s", backup.path)
            else:
                LOG.info("Purging: %s", backup.path)
                backup.purge()
        LOG.info(
            "%s now has %s of available space",
            os.path.join(self.spool.path, name),
            format_bytes(disk_free(os.path.join(self.spool.path, name))),
        )
        return True
Example #9
0
 def estimate_backup_size(self):
     """Estimate the size of the next backup"""
     try:
         client = MySQL.from_defaults(self.defaults_path)
     except MySQL.MySQLError as exc:
         raise BackupError("Failed to connect to MySQL [%d] %s" % exc.args)
     try:
         try:
             datadir = client.var("datadir")
             return directory_size(datadir)
         except MySQL.MySQLError as exc:
             raise BackupError("Failed to find mysql datadir: [%d] %s" % exc.args)
         except OSError as exc:
             raise BackupError(
                 "Failed to calculate directory size: [%d] %s" % (exc.errno, exc.strerror)
             )
     finally:
         client.close()
Example #10
0
 def estimate_backup_size(self):
     try:
         client = MySQL.from_defaults(self.defaults_path)
     except MySQL.MySQLError as exc:
         raise BackupError('Failed to connect to MySQL [%d] %s' % exc.args)
     try:
         try:
             datadir = client.var('datadir')
             return directory_size(datadir)
         except MySQL.MySQLError as exc:
             raise BackupError("Failed to find mysql datadir: [%d] %s" %
                               exc.args)
         except OSError as exc:
             raise BackupError(
                 'Failed to calculate directory size: [%d] %s' %
                 (exc.errno, exc.strerror))
     finally:
         client.close()
Example #11
0
class XtrabackupPlugin(object):
    #: control connection to mysql server
    mysql = None

    #: path to the my.cnf generated by this plugin
    defaults_path = None

    def __init__(self, name, config, target_directory, dry_run=False):
        self.name = name
        self.config = config
        self.config.validate_config(CONFIGSPEC)
        self.target_directory = target_directory
        self.dry_run = dry_run

        defaults_path = join(self.target_directory, 'my.cnf')
        client_opts = self.config['mysql:client']
        includes = client_opts['defaults-extra-file'] + \
                   [self.config['xtrabackup']['global-defaults']]
        util.generate_defaults_file(defaults_path, includes, client_opts)
        self.defaults_path = defaults_path

    def estimate_backup_size(self):
        try:
            client = MySQL.from_defaults(self.defaults_path)
        except MySQL.MySQLError, exc:
            raise BackupError('Failed to connect to MySQL [%d] %s' % exc.args)
        try:
            try:
                datadir = client.var('datadir')
                return directory_size(datadir)
            except MySQL.MySQLError, exc:
                raise BackupError("Failed to find mysql datadir: [%d] %s" %
                                  exc.args)
            except OSError, exc:
                raise BackupError(
                    'Failed to calculate directory size: [%d] %s' %
                    (exc.errno, exc.strerror))
Example #12
0
        if not isinstance(exc, BackupError):
            LOGGER.debug("Unexpected exception when running backups.",
                         exc_info=True)
            exc = BackupError(exc)

    backup_job.config['holland:backup']['stop-time'] = time.time()
    backup_interval = (backup_job.config['holland:backup']['stop-time'] -
                       backup_job.config['holland:backup']['start-time'])
    if dry_run:
        LOGGER.info("Dry-run completed in %s",
                    format_interval(backup_interval))
    else:
        LOGGER.info("Backup completed in %s", format_interval(backup_interval))

    if not dry_run and exc is None:
        final_size = directory_size(backup_job.path)
        LOGGER.info(
            "Final on-disk backup size: %s %.2f%% of estimated size %s",
            format_bytes(final_size),
            estimated_size and 100 * (float(final_size) / estimated_size)
            or 0.0, format_bytes(estimated_size))
        backup_job.config['holland:backup']['on-disk-size'] = final_size
        LOGGER.debug("Flushing backup job")
        backup_job.flush()

    if exc is not None:
        if backup_job.config['holland:backup']['auto-purge-failures'] is True:
            LOGGER.warning("Purging this backup (%s) due to failure",
                           backup_job.name)
            backup_job.purge()
        raise
Example #13
0
    def backup(self, name, config, dry_run=False):
        """Run a backup for the named backupset using the provided
        configuration

        :param name: name of the backupset
        :param config: dict-like object providing the backupset configuration

        :raises: BackupError if a backup fails
        """
        for i in range(MAX_SPOOL_RETRIES):
            try:
                spool_entry = self.spool.add_backup(name)
                break
            except OSError as exc:
                if exc.errno != errno.EEXIST:
                    raise BackupError("Failed to create spool: %s" % exc)
                LOG.debug("Failed to create spool.  Retrying in %d seconds.", i + 1)
                time.sleep(i + 1)
        else:
            raise BackupError("Failed to create a new backup directory for %s" % name)

        spool_entry.config.merge(config)
        spool_entry.validate_config()

        if dry_run:
            # always purge the spool
            self.register_cb("post-backup", lambda *args, **kwargs: spool_entry.purge())

        plugin = load_plugin(name, spool_entry.config, spool_entry.path, dry_run)

        spool_entry.config["holland:backup"]["start-time"] = time.time()
        spool_entry.flush()
        self.apply_cb("before-backup", spool_entry)
        spool_entry.config["holland:backup"]["failed"] = False

        try:
            estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
            LOG.info(
                "Starting backup[%s] via plugin %s",
                spool_entry.name,
                spool_entry.config["holland:backup"]["plugin"],
            )
            plugin.backup()
        except KeyboardInterrupt:
            LOG.warning("Backup aborted by interrupt")
            spool_entry.config["holland:backup"]["failed"] = True
            raise
        except BaseException as ex:
            LOG.warning(ex)
            spool_entry.config["holland:backup"]["failed"] = True

        spool_entry.config["holland:backup"]["stop-time"] = time.time()
        if not dry_run and not spool_entry.config["holland:backup"]["failed"]:
            final_size = float(directory_size(spool_entry.path))
            LOG.info("Final on-disk backup size %s", format_bytes(final_size))
            if estimated_size > 0:
                LOG.info(
                    "%.2f%% of estimated size %s",
                    (final_size / estimated_size) * 100.0,
                    format_bytes(estimated_size),
                )

            spool_entry.config["holland:backup"]["on-disk-size"] = final_size
            spool_entry.flush()

        start_time = spool_entry.config["holland:backup"]["start-time"]
        stop_time = spool_entry.config["holland:backup"]["stop-time"]

        if spool_entry.config["holland:backup"]["failed"]:
            LOG.error("Backup failed after %s", format_interval(stop_time - start_time))
        else:
            LOG.info("Backup completed in %s", format_interval(stop_time - start_time))

        if dry_run:
            spool_entry.purge()

        if sys.exc_info() != (None, None, None) or spool_entry.config["holland:backup"]["failed"]:
            LOG.debug("sys.exc_info(): %r", sys.exc_info())
            self.apply_cb("failed-backup", spool_entry)
            raise BackupError("Failed backup: %s" % name)
        else:
            self.apply_cb("after-backup", spool_entry)
Example #14
0
    except Exception, exc:
        if not isinstance(exc, BackupError):
            LOGGER.debug("Unexpected exception when running backups.", exc_info=True)
            exc = BackupError(exc)

    backup_job.config['holland:backup']['stop-time'] = time.time()
    backup_interval = (backup_job.config['holland:backup']['stop-time'] -
                       backup_job.config['holland:backup']['start-time'])
    if dry_run:
        LOGGER.info("Dry-run completed in %s",
                    format_interval(backup_interval))
    else:
        LOGGER.info("Backup completed in %s", 
                    format_interval(backup_interval))

    if not dry_run and exc is None:
        final_size = directory_size(backup_job.path)
        LOGGER.info("Final on-disk backup size: %s %.2f%% of estimated size %s", 
                    format_bytes(final_size), 
                    estimated_size and 100*(float(final_size)/estimated_size) or 0.0,
                    format_bytes(estimated_size))
        backup_job.config['holland:backup']['on-disk-size'] = final_size
        LOGGER.debug("Flushing backup job")
        backup_job.flush()

    if exc is not None:
        if backup_job.config['holland:backup']['auto-purge-failures'] is True:
            LOGGER.warning("Purging this backup (%s) due to failure", backup_job.name)
            backup_job.purge()
        raise
Example #15
0
            estimated_size = self.check_available_space(plugin, dry_run)
            LOG.info("Starting backup[%s] via plugin %s",
                     spool_entry.name,
                     spool_entry.config['holland:backup']['plugin'])
            plugin.backup()
        except KeyboardInterrupt:
            LOG.warning("Backup aborted by interrupt")
            spool_entry.config['holland:backup']['failed'] = True
        except:
            spool_entry.config['holland:backup']['failed'] = True
        else:
            spool_entry.config['holland:backup']['failed'] = False

        spool_entry.config['holland:backup']['stop-time'] = time.time()
        if not dry_run and not spool_entry.config['holland:backup']['failed']:
            final_size = directory_size(spool_entry.path)
            LOG.info("Final on-disk backup size %s", format_bytes(final_size))
            if estimated_size > 0:
                LOG.info("%.2f%% of estimated size %s",
                     (float(final_size) / estimated_size)*100.0,
                     format_bytes(estimated_size))

            spool_entry.config['holland:backup']['on-disk-size'] = final_size
            spool_entry.flush()

        start_time = spool_entry.config['holland:backup']['start-time']
        stop_time = spool_entry.config['holland:backup']['stop-time']

        if spool_entry.config['holland:backup']['failed']:
            LOG.error("Backup failed after %s",
                      format_interval(stop_time - start_time))
Example #16
0
    def backup(self, name, config, dry_run=False):
        """Run a backup for the named backupset using the provided
        configuration

        :param name: name of the backupset
        :param config: dict-like object providing the backupset configuration

        :raises: BackupError if a backup fails
        """
        for i in range(MAX_SPOOL_RETRIES):
            try:
                spool_entry = self.spool.add_backup(name)
                break
            except OSError as exc:
                if exc.errno != errno.EEXIST:
                    raise BackupError("Failed to create spool: %s" % exc)
                LOG.debug("Failed to create spool.  Retrying in %d seconds.", i + 1)
                time.sleep(i + 1)
        else:
            raise BackupError("Failed to create a new backup directory for %s" % name)

        spool_entry.config.merge(config)
        spool_entry.validate_config()

        if dry_run:
            # always purge the spool
            self.register_cb("post-backup", lambda *args, **kwargs: spool_entry.purge())

        plugin = load_plugin(name, spool_entry.config, spool_entry.path, dry_run)

        spool_entry.config["holland:backup"]["start-time"] = time.time()
        spool_entry.flush()
        self.apply_cb("before-backup", spool_entry)
        spool_entry.config["holland:backup"]["failed"] = False

        try:
            estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
            LOG.info(
                "Starting backup[%s] via plugin %s",
                spool_entry.name,
                spool_entry.config["holland:backup"]["plugin"],
            )
            plugin.backup()
        except KeyboardInterrupt:
            LOG.warning("Backup aborted by interrupt")
            spool_entry.config["holland:backup"]["failed"] = True
            raise
        except BaseException as ex:
            LOG.warning(ex)
            spool_entry.config["holland:backup"]["failed"] = True

        spool_entry.config["holland:backup"]["stop-time"] = time.time()
        if not dry_run and not spool_entry.config["holland:backup"]["failed"]:
            final_size = float(directory_size(spool_entry.path))
            LOG.info("Final on-disk backup size %s", format_bytes(final_size))
            if estimated_size > 0:
                LOG.info(
                    "%.2f%% of estimated size %s",
                    (final_size / estimated_size) * 100.0,
                    format_bytes(estimated_size),
                )

            spool_entry.config["holland:backup"]["on-disk-size"] = final_size
            spool_entry.flush()

        start_time = spool_entry.config["holland:backup"]["start-time"]
        stop_time = spool_entry.config["holland:backup"]["stop-time"]

        if spool_entry.config["holland:backup"]["failed"]:
            LOG.error("Backup failed after %s", format_interval(stop_time - start_time))
        else:
            LOG.info("Backup completed in %s", format_interval(stop_time - start_time))

        if dry_run:
            spool_entry.purge()

        if sys.exc_info() != (None, None, None) or spool_entry.config["holland:backup"]["failed"]:
            LOG.debug("sys.exc_info(): %r", sys.exc_info())
            self.apply_cb("failed-backup", spool_entry)
            raise BackupError("Failed backup: %s" % name)
        else:
            self.apply_cb("after-backup", spool_entry)
Example #17
0
            estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
            LOG.info("Starting backup[%s] via plugin %s",
                     spool_entry.name,
                     spool_entry.config['holland:backup']['plugin'])
            plugin.backup()
        except KeyboardInterrupt:
            LOG.warning("Backup aborted by interrupt")
            spool_entry.config['holland:backup']['failed'] = True
        except:
            spool_entry.config['holland:backup']['failed'] = True
        else:
            spool_entry.config['holland:backup']['failed'] = False

        spool_entry.config['holland:backup']['stop-time'] = time.time()
        if not dry_run and not spool_entry.config['holland:backup']['failed']:
            final_size = float(directory_size(spool_entry.path))
            LOG.info("Final on-disk backup size %s", format_bytes(final_size))
            if estimated_size > 0:
                LOG.info("%.2f%% of estimated size %s",
                     (final_size / estimated_size)*100.0,
                     format_bytes(estimated_size))

            spool_entry.config['holland:backup']['on-disk-size'] = final_size
            spool_entry.flush()

        start_time = spool_entry.config['holland:backup']['start-time']
        stop_time = spool_entry.config['holland:backup']['stop-time']

        if spool_entry.config['holland:backup']['failed']:
            LOG.error("Backup failed after %s",
                      format_interval(stop_time - start_time))
Example #18
0
            estimated_size = self.check_available_space(plugin)
            LOG.info("Starting backup[%s] via plugin %s",
                     spool_entry.name,
                     spool_entry.config['holland:backup']['plugin'])
            plugin.backup()
        except KeyboardInterrupt:
            LOG.warning("Backup aborted by interrupt")
            spool_entry.config['holland:backup']['failed'] = True
        except:
            spool_entry.config['holland:backup']['failed'] = True
        else:
            spool_entry.config['holland:backup']['failed'] = False

        spool_entry.config['holland:backup']['stop-time'] = time.time()
        if not dry_run and not spool_entry.config['holland:backup']['failed']:
            final_size = directory_size(spool_entry.path)
            LOG.info("Final on-disk backup size %s", format_bytes(final_size))
            if estimated_size > 0:
                LOG.info("%.2f%% of estimated size %s",
                     (float(final_size) / estimated_size)*100.0,
                     format_bytes(estimated_size))

            spool_entry.config['holland:backup']['on-disk-size'] = final_size
            spool_entry.flush()

        start_time = spool_entry.config['holland:backup']['start-time']
        stop_time = spool_entry.config['holland:backup']['stop-time']

        if spool_entry.config['holland:backup']['failed']:
            LOG.error("Backup failed after %s",
                      format_interval(stop_time - start_time))