class MysqlLVMBackup(object):
"""A Holland Backup plugin suitable for performing LVM snapshots of a
filesystem underlying a live MySQL instance.
This plugin produces tar archives of a MySQL data directory.
"""
CONFIGSPEC = CONFIGSPEC
def __init__(self, name, config, target_directory, dry_run=False):
self.config = config
self.config.validate_config(self.configspec())
LOG.debug("Validated config: %r", self.config)
self.name = name
self.target_directory = target_directory
self.dry_run = dry_run
self.client = connect_simple(self.config['mysql:client'])
def estimate_backup_size(self):
"""Estimate the backup size this plugin will produce
This is currently the total directory size of the MySQL datadir
"""
try:
self.client.connect()
datadir = self.client.show_variable('datadir')
self.client.disconnect()
except MySQLError, exc:
raise BackupError("[%d] %s" % exc.args)
return directory_size(datadir)
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will produce"""
try:
mysql_config = build_mysql_config(self.config['mysql:client'])
client = connect(mysql_config['client'])
datadir = client.show_variable('datadir')
return directory_size(datadir)
except MySQLError, exc:
raise BackupError("Failed to lookup the MySQL datadir when "
"estimating backup size: [%d] %s" % exc.args)
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will produce"""
try:
mysql_config = build_mysql_config(self.config['mysql:client'])
client = connect(mysql_config['client'])
datadir = client.show_variable('datadir')
return directory_size(datadir)
except MySQLError, exc:
raise BackupError("Failed to lookup the MySQL datadir when "
"estimating backup size: [%d] %s" % exc.args)
def free_required_space(self, name, required_bytes, dry_run=False):
"""Attempt to free at least ``required_bytes`` of old backups from a backupset
:param name: name of the backupset to free space from
:param required_bytes: integer number of bytes required for the backupset path
:param dry_run: if true, this will only generate log messages but won't actually free space
:returns: bool; True if freed or False otherwise
"""
LOG.info(
"Insufficient disk space for adjusted estimated backup size: %s",
format_bytes(required_bytes),
)
LOG.info(
"purge-on-demand is enabled. Discovering old backups to purge.")
available_bytes = disk_free(os.path.join(self.spool.path, name))
to_purge = {}
for backup in self.spool.list_backups(name):
backup_size = directory_size(backup.path)
LOG.info("Found backup '%s': %s", backup.path,
format_bytes(backup_size))
available_bytes += backup_size
to_purge[backup] = backup_size
if available_bytes > required_bytes:
break
else:
LOG.info("Purging would only recover an additional %s",
format_bytes(sum(to_purge.values())))
LOG.info(
"Only %s total would be available, but the current "
"backup requires %s",
format_bytes(available_bytes),
format_bytes(required_bytes),
)
return False
purge_bytes = sum(to_purge.values())
LOG.info(
"Found %d backups to purge which will recover %s",
len(to_purge),
format_bytes(purge_bytes),
)
for backup in to_purge:
if dry_run:
LOG.info("Would purge: %s", backup.path)
else:
LOG.info("Purging: %s", backup.path)
backup.purge()
LOG.info(
"%s now has %s of available space",
os.path.join(self.spool.path, name),
format_bytes(disk_free(os.path.join(self.spool.path, name))),
)
return True
def estimate_backup_size(self):
"""Estimate the backup size this plugin will produce
This is currently the total directory size of the MySQL datadir
"""
try:
self.client.connect()
datadir = self.client.show_variable("datadir")
self.client.disconnect()
except MySQLError as exc:
raise BackupError("[%d] %s" % exc.args)
return directory_size(datadir)
def estimate_backup_size(self):
"""Return estimated backup size"""
mysql_config = build_mysql_config(self.config["mysql:client"])
client = connect(mysql_config["client"])
try:
datadir = client.show_variable("datadir")
return directory_size(datadir)
except OSError as exc:
raise BackupError("Failed to calculate directory size: [%d] %s" %
(exc.errno, exc.strerror))
finally:
client.close()
def estimate_backup_size(self):
"""Estimate the backup size this plugin will produce
This is currently the total directory size of the MySQL datadir
"""
try:
self.client.connect()
datadir = self.client.show_variable('datadir')
self.client.disconnect()
except MySQLError as exc:
raise BackupError("[%d] %s" % exc.args)
return directory_size(datadir)
def free_required_space(self, name, required_bytes, dry_run=False):
"""Attempt to free at least ``required_bytes`` of old backups from a backupset
:param name: name of the backupset to free space from
:param required_bytes: integer number of bytes required for the backupset path
:param dry_run: if true, this will only generate log messages but won't actually free space
:returns: bool; True if freed or False otherwise
"""
LOG.info(
"Insufficient disk space for adjusted estimated backup size: %s",
format_bytes(required_bytes),
)
LOG.info("purge-on-demand is enabled. Discovering old backups to purge.")
available_bytes = disk_free(os.path.join(self.spool.path, name))
to_purge = {}
for backup in self.spool.list_backups(name):
backup_size = directory_size(backup.path)
LOG.info("Found backup '%s': %s", backup.path, format_bytes(backup_size))
available_bytes += backup_size
to_purge[backup] = backup_size
if available_bytes > required_bytes:
break
else:
LOG.info(
"Purging would only recover an additional %s", format_bytes(sum(to_purge.values()))
)
LOG.info(
"Only %s total would be available, but the current " "backup requires %s",
format_bytes(available_bytes),
format_bytes(required_bytes),
)
return False
purge_bytes = sum(to_purge.values())
LOG.info(
"Found %d backups to purge which will recover %s",
len(to_purge),
format_bytes(purge_bytes),
)
for backup in to_purge:
if dry_run:
LOG.info("Would purge: %s", backup.path)
else:
LOG.info("Purging: %s", backup.path)
backup.purge()
LOG.info(
"%s now has %s of available space",
os.path.join(self.spool.path, name),
format_bytes(disk_free(os.path.join(self.spool.path, name))),
)
return True
def estimate_backup_size(self):
"""Estimate the size of the next backup"""
try:
client = MySQL.from_defaults(self.defaults_path)
except MySQL.MySQLError as exc:
raise BackupError("Failed to connect to MySQL [%d] %s" % exc.args)
try:
try:
datadir = client.var("datadir")
return directory_size(datadir)
except MySQL.MySQLError as exc:
raise BackupError("Failed to find mysql datadir: [%d] %s" % exc.args)
except OSError as exc:
raise BackupError(
"Failed to calculate directory size: [%d] %s" % (exc.errno, exc.strerror)
)
finally:
client.close()
def estimate_backup_size(self):
try:
client = MySQL.from_defaults(self.defaults_path)
except MySQL.MySQLError as exc:
raise BackupError('Failed to connect to MySQL [%d] %s' % exc.args)
try:
try:
datadir = client.var('datadir')
return directory_size(datadir)
except MySQL.MySQLError as exc:
raise BackupError("Failed to find mysql datadir: [%d] %s" %
exc.args)
except OSError as exc:
raise BackupError(
'Failed to calculate directory size: [%d] %s' %
(exc.errno, exc.strerror))
finally:
client.close()
class XtrabackupPlugin(object):
#: control connection to mysql server
mysql = None
#: path to the my.cnf generated by this plugin
defaults_path = None
def __init__(self, name, config, target_directory, dry_run=False):
self.name = name
self.config = config
self.config.validate_config(CONFIGSPEC)
self.target_directory = target_directory
self.dry_run = dry_run
defaults_path = join(self.target_directory, 'my.cnf')
client_opts = self.config['mysql:client']
includes = client_opts['defaults-extra-file'] + \
[self.config['xtrabackup']['global-defaults']]
util.generate_defaults_file(defaults_path, includes, client_opts)
self.defaults_path = defaults_path
def estimate_backup_size(self):
try:
client = MySQL.from_defaults(self.defaults_path)
except MySQL.MySQLError, exc:
raise BackupError('Failed to connect to MySQL [%d] %s' % exc.args)
try:
try:
datadir = client.var('datadir')
return directory_size(datadir)
except MySQL.MySQLError, exc:
raise BackupError("Failed to find mysql datadir: [%d] %s" %
exc.args)
except OSError, exc:
raise BackupError(
'Failed to calculate directory size: [%d] %s' %
(exc.errno, exc.strerror))
if not isinstance(exc, BackupError):
LOGGER.debug("Unexpected exception when running backups.",
exc_info=True)
exc = BackupError(exc)
backup_job.config['holland:backup']['stop-time'] = time.time()
backup_interval = (backup_job.config['holland:backup']['stop-time'] -
backup_job.config['holland:backup']['start-time'])
if dry_run:
LOGGER.info("Dry-run completed in %s",
format_interval(backup_interval))
else:
LOGGER.info("Backup completed in %s", format_interval(backup_interval))
if not dry_run and exc is None:
final_size = directory_size(backup_job.path)
LOGGER.info(
"Final on-disk backup size: %s %.2f%% of estimated size %s",
format_bytes(final_size),
estimated_size and 100 * (float(final_size) / estimated_size)
or 0.0, format_bytes(estimated_size))
backup_job.config['holland:backup']['on-disk-size'] = final_size
LOGGER.debug("Flushing backup job")
backup_job.flush()
if exc is not None:
if backup_job.config['holland:backup']['auto-purge-failures'] is True:
LOGGER.warning("Purging this backup (%s) due to failure",
backup_job.name)
backup_job.purge()
raise
def backup(self, name, config, dry_run=False):
"""Run a backup for the named backupset using the provided
configuration
:param name: name of the backupset
:param config: dict-like object providing the backupset configuration
:raises: BackupError if a backup fails
"""
for i in range(MAX_SPOOL_RETRIES):
try:
spool_entry = self.spool.add_backup(name)
break
except OSError as exc:
if exc.errno != errno.EEXIST:
raise BackupError("Failed to create spool: %s" % exc)
LOG.debug("Failed to create spool. Retrying in %d seconds.", i + 1)
time.sleep(i + 1)
else:
raise BackupError("Failed to create a new backup directory for %s" % name)
spool_entry.config.merge(config)
spool_entry.validate_config()
if dry_run:
# always purge the spool
self.register_cb("post-backup", lambda *args, **kwargs: spool_entry.purge())
plugin = load_plugin(name, spool_entry.config, spool_entry.path, dry_run)
spool_entry.config["holland:backup"]["start-time"] = time.time()
spool_entry.flush()
self.apply_cb("before-backup", spool_entry)
spool_entry.config["holland:backup"]["failed"] = False
try:
estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
LOG.info(
"Starting backup[%s] via plugin %s",
spool_entry.name,
spool_entry.config["holland:backup"]["plugin"],
)
plugin.backup()
except KeyboardInterrupt:
LOG.warning("Backup aborted by interrupt")
spool_entry.config["holland:backup"]["failed"] = True
raise
except BaseException as ex:
LOG.warning(ex)
spool_entry.config["holland:backup"]["failed"] = True
spool_entry.config["holland:backup"]["stop-time"] = time.time()
if not dry_run and not spool_entry.config["holland:backup"]["failed"]:
final_size = float(directory_size(spool_entry.path))
LOG.info("Final on-disk backup size %s", format_bytes(final_size))
if estimated_size > 0:
LOG.info(
"%.2f%% of estimated size %s",
(final_size / estimated_size) * 100.0,
format_bytes(estimated_size),
)
spool_entry.config["holland:backup"]["on-disk-size"] = final_size
spool_entry.flush()
start_time = spool_entry.config["holland:backup"]["start-time"]
stop_time = spool_entry.config["holland:backup"]["stop-time"]
if spool_entry.config["holland:backup"]["failed"]:
LOG.error("Backup failed after %s", format_interval(stop_time - start_time))
else:
LOG.info("Backup completed in %s", format_interval(stop_time - start_time))
if dry_run:
spool_entry.purge()
if sys.exc_info() != (None, None, None) or spool_entry.config["holland:backup"]["failed"]:
LOG.debug("sys.exc_info(): %r", sys.exc_info())
self.apply_cb("failed-backup", spool_entry)
raise BackupError("Failed backup: %s" % name)
else:
self.apply_cb("after-backup", spool_entry)
except Exception, exc:
if not isinstance(exc, BackupError):
LOGGER.debug("Unexpected exception when running backups.", exc_info=True)
exc = BackupError(exc)
backup_job.config['holland:backup']['stop-time'] = time.time()
backup_interval = (backup_job.config['holland:backup']['stop-time'] -
backup_job.config['holland:backup']['start-time'])
if dry_run:
LOGGER.info("Dry-run completed in %s",
format_interval(backup_interval))
else:
LOGGER.info("Backup completed in %s",
format_interval(backup_interval))
if not dry_run and exc is None:
final_size = directory_size(backup_job.path)
LOGGER.info("Final on-disk backup size: %s %.2f%% of estimated size %s",
format_bytes(final_size),
estimated_size and 100*(float(final_size)/estimated_size) or 0.0,
format_bytes(estimated_size))
backup_job.config['holland:backup']['on-disk-size'] = final_size
LOGGER.debug("Flushing backup job")
backup_job.flush()
if exc is not None:
if backup_job.config['holland:backup']['auto-purge-failures'] is True:
LOGGER.warning("Purging this backup (%s) due to failure", backup_job.name)
backup_job.purge()
raise
estimated_size = self.check_available_space(plugin, dry_run)
LOG.info("Starting backup[%s] via plugin %s",
spool_entry.name,
spool_entry.config['holland:backup']['plugin'])
plugin.backup()
except KeyboardInterrupt:
LOG.warning("Backup aborted by interrupt")
spool_entry.config['holland:backup']['failed'] = True
except:
spool_entry.config['holland:backup']['failed'] = True
else:
spool_entry.config['holland:backup']['failed'] = False
spool_entry.config['holland:backup']['stop-time'] = time.time()
if not dry_run and not spool_entry.config['holland:backup']['failed']:
final_size = directory_size(spool_entry.path)
LOG.info("Final on-disk backup size %s", format_bytes(final_size))
if estimated_size > 0:
LOG.info("%.2f%% of estimated size %s",
(float(final_size) / estimated_size)*100.0,
format_bytes(estimated_size))
spool_entry.config['holland:backup']['on-disk-size'] = final_size
spool_entry.flush()
start_time = spool_entry.config['holland:backup']['start-time']
stop_time = spool_entry.config['holland:backup']['stop-time']
if spool_entry.config['holland:backup']['failed']:
LOG.error("Backup failed after %s",
format_interval(stop_time - start_time))
def backup(self, name, config, dry_run=False):
"""Run a backup for the named backupset using the provided
configuration
:param name: name of the backupset
:param config: dict-like object providing the backupset configuration
:raises: BackupError if a backup fails
"""
for i in range(MAX_SPOOL_RETRIES):
try:
spool_entry = self.spool.add_backup(name)
break
except OSError as exc:
if exc.errno != errno.EEXIST:
raise BackupError("Failed to create spool: %s" % exc)
LOG.debug("Failed to create spool. Retrying in %d seconds.", i + 1)
time.sleep(i + 1)
else:
raise BackupError("Failed to create a new backup directory for %s" % name)
spool_entry.config.merge(config)
spool_entry.validate_config()
if dry_run:
# always purge the spool
self.register_cb("post-backup", lambda *args, **kwargs: spool_entry.purge())
plugin = load_plugin(name, spool_entry.config, spool_entry.path, dry_run)
spool_entry.config["holland:backup"]["start-time"] = time.time()
spool_entry.flush()
self.apply_cb("before-backup", spool_entry)
spool_entry.config["holland:backup"]["failed"] = False
try:
estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
LOG.info(
"Starting backup[%s] via plugin %s",
spool_entry.name,
spool_entry.config["holland:backup"]["plugin"],
)
plugin.backup()
except KeyboardInterrupt:
LOG.warning("Backup aborted by interrupt")
spool_entry.config["holland:backup"]["failed"] = True
raise
except BaseException as ex:
LOG.warning(ex)
spool_entry.config["holland:backup"]["failed"] = True
spool_entry.config["holland:backup"]["stop-time"] = time.time()
if not dry_run and not spool_entry.config["holland:backup"]["failed"]:
final_size = float(directory_size(spool_entry.path))
LOG.info("Final on-disk backup size %s", format_bytes(final_size))
if estimated_size > 0:
LOG.info(
"%.2f%% of estimated size %s",
(final_size / estimated_size) * 100.0,
format_bytes(estimated_size),
)
spool_entry.config["holland:backup"]["on-disk-size"] = final_size
spool_entry.flush()
start_time = spool_entry.config["holland:backup"]["start-time"]
stop_time = spool_entry.config["holland:backup"]["stop-time"]
if spool_entry.config["holland:backup"]["failed"]:
LOG.error("Backup failed after %s", format_interval(stop_time - start_time))
else:
LOG.info("Backup completed in %s", format_interval(stop_time - start_time))
if dry_run:
spool_entry.purge()
if sys.exc_info() != (None, None, None) or spool_entry.config["holland:backup"]["failed"]:
LOG.debug("sys.exc_info(): %r", sys.exc_info())
self.apply_cb("failed-backup", spool_entry)
raise BackupError("Failed backup: %s" % name)
else:
self.apply_cb("after-backup", spool_entry)
estimated_size = self.check_available_space(plugin, spool_entry, dry_run)
LOG.info("Starting backup[%s] via plugin %s",
spool_entry.name,
spool_entry.config['holland:backup']['plugin'])
plugin.backup()
except KeyboardInterrupt:
LOG.warning("Backup aborted by interrupt")
spool_entry.config['holland:backup']['failed'] = True
except:
spool_entry.config['holland:backup']['failed'] = True
else:
spool_entry.config['holland:backup']['failed'] = False
spool_entry.config['holland:backup']['stop-time'] = time.time()
if not dry_run and not spool_entry.config['holland:backup']['failed']:
final_size = float(directory_size(spool_entry.path))
LOG.info("Final on-disk backup size %s", format_bytes(final_size))
if estimated_size > 0:
LOG.info("%.2f%% of estimated size %s",
(final_size / estimated_size)*100.0,
format_bytes(estimated_size))
spool_entry.config['holland:backup']['on-disk-size'] = final_size
spool_entry.flush()
start_time = spool_entry.config['holland:backup']['start-time']
stop_time = spool_entry.config['holland:backup']['stop-time']
if spool_entry.config['holland:backup']['failed']:
LOG.error("Backup failed after %s",
format_interval(stop_time - start_time))
estimated_size = self.check_available_space(plugin)
LOG.info("Starting backup[%s] via plugin %s",
spool_entry.name,
spool_entry.config['holland:backup']['plugin'])
plugin.backup()
except KeyboardInterrupt:
LOG.warning("Backup aborted by interrupt")
spool_entry.config['holland:backup']['failed'] = True
except:
spool_entry.config['holland:backup']['failed'] = True
else:
spool_entry.config['holland:backup']['failed'] = False
spool_entry.config['holland:backup']['stop-time'] = time.time()
if not dry_run and not spool_entry.config['holland:backup']['failed']:
final_size = directory_size(spool_entry.path)
LOG.info("Final on-disk backup size %s", format_bytes(final_size))
if estimated_size > 0:
LOG.info("%.2f%% of estimated size %s",
(float(final_size) / estimated_size)*100.0,
format_bytes(estimated_size))
spool_entry.config['holland:backup']['on-disk-size'] = final_size
spool_entry.flush()
start_time = spool_entry.config['holland:backup']['start-time']
stop_time = spool_entry.config['holland:backup']['stop-time']
if spool_entry.config['holland:backup']['failed']:
LOG.error("Backup failed after %s",
format_interval(stop_time - start_time))