def _stop_slave(client, config):
"""Stop MySQL replication"""
try:
client.stop_slave(sql_thread_only=True)
LOG.info("Stopped slave")
config['mysql:replication'] = {}
repl_cfg = config['mysql:replication']
except MySQLError as exc:
raise BackupError("Failed to stop slave[%d]: %s" % exc.args)
try:
slave_info = client.show_slave_status()
if slave_info:
# update config with replication info
log_file = slave_info['relay_master_log_file']
log_pos = slave_info['exec_master_log_pos']
repl_cfg['slave_master_log_file'] = log_file
repl_cfg['slave_master_log_pos'] = log_pos
except MySQLError as exc:
raise BackupError("Failed to acquire slave status[%d]: %s" % \
exc.args)
try:
master_info = client.show_master_status()
if master_info:
repl_cfg['master_log_file'] = master_info['file']
repl_cfg['master_log_pos'] = master_info['position']
except MySQLError as exc:
raise BackupError("Failed to acquire master status [%d] %s" % exc.args)
LOG.info("MySQL Replication has been stopped.")
def check(self):
LOG.info("Checking that SQLite backups can run.")
if not os.path.exists(self.sqlite_bin):
raise BackupError("SQLite binary [%s] doesn't exist!" %
self.sqlite_bin)
for db in self.config['sqlite']['databases']:
# sometimes picks up empty string ('')
if not db:
continue
path = os.path.abspath(os.path.expanduser(db))
if not os.path.exists(path):
LOG.error("SQLite database [%s] doesn't exist!" % path)
self.invalid_databases.append(db)
continue
process = Popen([self.sqlite_bin, path, '.schema'],
stdin=open('/dev/null', 'r'),
stdout=open('/dev/null', 'w'),
stderr=PIPE)
_, stderroutput = process.communicate()
if process.returncode != 0:
LOG.error(stderroutput)
self.invalid_databases.append(db)
else:
self.databases.append(db)
if len(self.databases) == 0 and len(self.invalid_databases) == 0:
raise BackupError("No SQLite databases to backup!")
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will generate"""
LOG.info("Estimating size of mysqldump backup")
estimate_method = self.config['mysqldump']['estimate-method']
if estimate_method.startswith('const:'):
try:
return parse_size(estimate_method[6:])
except ValueError as exc:
raise BackupError(str(exc))
if estimate_method != 'plugin':
raise BackupError("Invalid estimate-method '%s'" % estimate_method)
try:
db_iter = DatabaseIterator(self.client)
tbl_iter = MetadataTableIterator(self.client)
try:
self.client.connect()
except Exception as ex:
LOG.error("Failed to connect to database")
LOG.error("%s", ex)
raise BackupError("MySQL Error %s" % ex)
try:
self.schema.refresh(db_iter=db_iter, tbl_iter=tbl_iter)
except MySQLError as exc:
LOG.error("Failed to estimate backup size")
LOG.error("[%d] %s", *exc.args)
raise BackupError("MySQL Error [%d] %s" % exc.args)
return float(sum([db.size for db in self.schema.databases]))
finally:
self.client.disconnect()
def backup(self):
if self.dry_run:
return
if not os.path.exists(self.config['tar']['directory']) \
or not os.path.isdir(self.config['tar']['directory']):
raise BackupError('{0} is not a directory!'.format(self.config['tar']['directory']))
out_name = "{0}.tar".format(
self.config['tar']['directory'].lstrip('/').replace('/', '_'))
outfile = os.path.join(self.target_directory, out_name)
args = ['tar', 'c', self.config['tar']['directory']]
errlog = TemporaryFile()
stream = self._open_stream(outfile, 'w')
LOG.info("Executing: %s", list2cmdline(args))
pid = Popen(
args,
stdout=stream.fileno(),
stderr=errlog.fileno(),
close_fds=True)
status = pid.wait()
try:
errlog.flush()
errlog.seek(0)
for line in errlog:
LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip())
finally:
errlog.close()
if status != 0:
raise BackupError('tar failed (status={0})'.format(status))
def backup(self):
"""
Start a backup.
"""
# estimate and setup has completed at this point
# so ensure the connection is closed - we will never reuse this
self.connection.close()
if self.dry_run:
# Very simply dry run information
# enough to know that:
# 1) We can connect to Postgres using pgpass data
# 2) The exact databases we would dump
dry_run(self.databases, self.config)
return
# First run a pg_dumpall -g and save the globals
# Then run a pg_dump for each database we find
backup_dir = os.path.join(self.target_directory, 'data')
# put everything in data/
try:
os.mkdir(backup_dir)
except OSError as exc:
raise BackupError("Failed to create backup directory %s" % backup_dir)
try:
backup_pgsql(backup_dir, self.config, self.databases)
except (OSError, PgError) as exc:
LOG.debug("Failed to backup Postgres. %s",
str(exc), exc_info=True)
raise BackupError(str(exc))
def _backup(self):
"""Real backup method. May raise BackupError exceptions"""
config = self.config['mysqldump']
# setup defaults_file with ignore-table exclusions
defaults_file = os.path.join(self.target_directory, 'my.cnf')
write_options(self.mysql_config, defaults_file)
if config['exclude-invalid-views']:
LOG.info("* Finding and excluding invalid views...")
definitions_path = os.path.join(self.target_directory,
'invalid_views.sql')
exclude_invalid_views(self.schema, self.client, definitions_path)
add_exclusions(self.schema, defaults_file)
# find the path to the mysqldump command
mysqldump_bin = find_mysqldump(path=config['mysql-binpath'])
LOG.info("Using mysqldump executable: %s", mysqldump_bin)
# setup the mysqldump environment
extra_defaults = config['extra-defaults']
try:
mysqldump = MySQLDump(defaults_file,
mysqldump_bin,
extra_defaults=extra_defaults)
except MySQLDumpError as exc:
raise BackupError(str(exc))
except Exception as ex:
LOG.warning(ex)
LOG.info("mysqldump version %s", '.'.join([str(digit)
for digit in mysqldump.version]))
options = collect_mysqldump_options(config, mysqldump, self.client)
validate_mysqldump_options(mysqldump, options)
os.mkdir(os.path.join(self.target_directory, 'backup_data'))
if self.config['compression']['method'] != 'none' and \
self.config['compression']['level'] > 0:
try:
cmd, ext = lookup_compression(self.config['compression']['method'])
except OSError as exc:
raise BackupError("Unable to load compression method '%s': %s" %
(self.config['compression']['method'], exc))
LOG.info("Using %s compression level %d with args %s",
self.config['compression']['method'],
self.config['compression']['level'],
self.config['compression']['options'])
else:
LOG.info("Not compressing mysqldump output")
cmd = ''
ext = ''
try:
start(mysqldump=mysqldump,
schema=self.schema,
lock_method=config['lock-method'],
file_per_database=config['file-per-database'],
open_stream=self._open_stream,
compression_ext=ext)
except MySQLDumpError as exc:
raise BackupError(str(exc))
def start(mysqldump,
schema=None,
lock_method='auto-detect',
file_per_database=True,
open_stream=open,
compression_ext=''):
"""Run a mysqldump backup"""
if not schema and file_per_database:
raise BackupError("file_per_database specified without a valid schema")
if not schema:
target_databases = ALL_DATABASES
else:
if len(schema.databases) == 0:
raise BackupError("No databases found to backup")
if not file_per_database and not [
x for x in schema.excluded_databases
]:
target_databases = ALL_DATABASES
else:
target_databases = [
db for db in schema.databases if not db.excluded
]
write_manifest(schema, open_stream, compression_ext)
if file_per_database:
flush_logs = '--flush-logs' in mysqldump.options
if flush_logs:
mysqldump.options.remove('--flush-logs')
last = len(target_databases)
for count, db in enumerate(target_databases):
more_options = [mysqldump_lock_option(lock_method, [db])]
# add --flush-logs only to the last mysqldump run
if flush_logs and count == last:
more_options.append('--flush-logs')
db_name = encode(db.name)[0]
if db_name != db.name:
LOG.warning("Encoding file-name for database %s to %s",
db.name, db_name)
try:
stream = open_stream('%s.sql' % db_name, 'w')
except (IOError, OSError), exc:
raise BackupError("Failed to open output stream %s: %s" %
('%s.sql' + compression_ext, str(exc)))
try:
mysqldump.run([db.name], stream, more_options)
finally:
try:
stream.close()
except (IOError, OSError), exc:
if exc.errno != errno.EPIPE:
LOG.error("%s", str(exc))
raise BackupError(str(exc))
class MySQLDumpPlugin(object):
"""MySQLDump Backup Plugin interface for Holland"""
CONFIGSPEC = CONFIGSPEC
def __init__(self, name, config, target_directory, dry_run=False):
self.name = name
self.config = config
self.target_directory = target_directory
self.dry_run = dry_run
self.config.validate_config(self.CONFIGSPEC) # -> ValidationError
# Setup a discovery shell to find schema items
# This will iterate over items during the estimate
# or backup phase, which will call schema.refresh()
self.schema = MySQLSchema()
config = self.config['mysqldump']
self.schema.add_database_filter(include_glob(*config['databases']))
self.schema.add_database_filter(
exclude_glob(*config['exclude-databases']))
self.schema.add_table_filter(include_glob_qualified(*config['tables']))
self.schema.add_table_filter(
exclude_glob_qualified(*config['exclude-tables']))
self.schema.add_engine_filter(include_glob(*config['engines']))
self.schema.add_engine_filter(exclude_glob(*config['exclude-engines']))
self.mysql_config = build_mysql_config(self.config['mysql:client'])
self.client = connect(self.mysql_config['client'])
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will generate"""
LOG.info("Estimating size of mysqldump backup")
estimate_method = self.config['mysqldump']['estimate-method']
if estimate_method.startswith('const:'):
try:
return parse_size(estimate_method[6:])
except ValueError, exc:
raise BackupError(str(exc))
if estimate_method != 'plugin':
raise BackupError("Invalid estimate-method '%s'" % estimate_method)
try:
db_iter = DatabaseIterator(self.client)
tbl_iter = MetadataTableIterator(self.client)
try:
self.client.connect()
self.schema.refresh(db_iter=db_iter, tbl_iter=tbl_iter)
except MySQLError, exc:
LOG.error("Failed to estimate backup size")
LOG.error("[%d] %s", *exc.args)
raise BackupError("MySQL Error [%d] %s" % exc.args)
return float(sum([db.size for db in self.schema.databases]))
def backup(self):
"""Run a backup by running through a LVM snapshot against the device
the MySQL datadir resides on
"""
# connect to mysql and lookup what we're supposed to snapshot
try:
self.client.connect()
datadir = os.path.realpath(self.client.show_variable('datadir'))
except MySQLError as exc:
raise BackupError("[%d] %s" % exc.args)
LOG.info("Backing up %s via snapshot", datadir)
# lookup the logical volume mysql's datadir sits on
try:
volume = LogicalVolume.lookup_from_fspath(datadir)
except LookupError as exc:
raise BackupError("Failed to lookup logical volume for %s: %s" %
(datadir, str(exc)))
except Exception as ex:
LOG.debug(ex)
# create a snapshot manager
snapshot = build_snapshot(self.config['mysql-lvm'], volume,
suppress_tmpdir=self.dry_run)
# calculate where the datadirectory on the snapshot will be located
rpath = relpath(datadir, getmount(datadir))
snap_datadir = os.path.abspath(os.path.join(snapshot.mountpoint, rpath))
# setup actions to perform at each step of the snapshot process
setup_actions(snapshot=snapshot,
config=self.config,
client=self.client,
snap_datadir=snap_datadir,
spooldir=self.target_directory)
if self.dry_run:
return self._dry_run(volume, snapshot, datadir)
try:
snapshot.start(volume)
except CallbackFailuresError as exc:
# XXX: one of our actions failed. Log this better
for callback, error in exc.errors:
LOG.error("%s", error)
raise BackupError("Error occurred during snapshot process. Aborting.")
except LVMCommandError as exc:
# Something failed in the snapshot process
raise BackupError(str(exc))
except Exception as ex:
LOG.debug(ex)
def backup(self):
"""Run a database backup with xtrabackup"""
defaults_file = os.path.join(self.target_directory,
'my.xtrabackup.cnf')
args = [
self.config['xtrabackup']['innobackupex'],
'--defaults-file=%s' % defaults_file,
'--stream=tar4ibd',
tempfile.gettempdir(),
]
if self.config['xtrabackup']['slave-info']:
args.insert(3, '--slave-info')
if self.config['xtrabackup']['no-lock']:
args.insert(2, '--no-lock')
LOG.info("%s", list2cmdline(args))
if self.dry_run:
return
config = build_mysql_config(self.config['mysql:client'])
write_options(config, defaults_file)
shutil.copyfileobj(
open(self.config['xtrabackup']['global-defaults'], 'r'),
open(defaults_file, 'a'))
backup_path = os.path.join(self.target_directory, 'backup.tar')
compression_stream = open_stream(backup_path, 'w',
**self.config['compression'])
error_log_path = os.path.join(self.target_directory, 'xtrabackup.log')
error_log = open(error_log_path, 'wb')
try:
try:
check_call(args,
stdout=compression_stream,
stderr=error_log,
close_fds=True)
except OSError, exc:
LOG.info("Command not found: %s", args[0])
raise BackupError("%s not found. Is xtrabackup installed?" %
args[0])
except CalledProcessError, exc:
LOG.info("%s failed", list2cmdline(exc.cmd))
for line in open(error_log_path, 'r'):
if line.startswith('>>'):
continue
LOG.info("%s", line.rstrip())
raise BackupError("%s failed" % exc.cmd[0])
def build_snapshot(config, logical_volume, suppress_tmpdir=False):
"""Create a snapshot process for running through the various steps
of creating, mounting, unmounting and removing a snapshot
"""
snapshot_name = config['snapshot-name'] or \
logical_volume.lv_name + '_snapshot'
extent_size = int(logical_volume.vg_extent_size)
snapshot_size = config['snapshot-size']
if not snapshot_size:
snapshot_size = min(
int(logical_volume.vg_free_count),
(int(logical_volume.lv_size) * 0.2) / extent_size,
(15 * 1024**3) / extent_size,
)
LOG.info("Auto-sizing snapshot-size to %s (%d extents)",
format_bytes(snapshot_size * extent_size), snapshot_size)
if snapshot_size < 1:
raise BackupError(
"Insufficient free extents on %s "
"to create snapshot (free extents = %s)" %
(logical_volume.device_name(), logical_volume.vg_free_count))
else:
try:
_snapshot_size = snapshot_size
snapshot_size = parse_bytes(snapshot_size) / extent_size
LOG.info(
"Using requested snapshot-size %s "
"rounded by extent-size %s to %s.", _snapshot_size,
format_bytes(extent_size),
format_bytes(snapshot_size * extent_size))
if snapshot_size < 1:
raise BackupError("Requested snapshot-size (%s) is "
"less than 1 extent" % _snapshot_size)
if snapshot_size > int(logical_volume.vg_free_count):
LOG.info(
"Snapshot size requested %s, but only %s available.",
config['snapshot-size'],
format_bytes(int(logical_volume.vg_free_count) *
extent_size,
precision=4))
LOG.info(
"Truncating snapshot-size to %d extents (%s)",
int(logical_volume.vg_free_count),
format_bytes(int(logical_volume.vg_free_count) *
extent_size,
precision=4))
snapshot_size = int(logical_volume.vg_free_count)
except ValueError, exc:
raise BackupError("Problem parsing snapshot-size %s" % exc)
def collect_mysqldump_options(config, mysqldump, client):
"""Do intelligent collection of mysqldump options from the config
and add any additional options for further validation"""
options = []
if config['flush-logs']:
options.append('--flush-logs')
if config['flush-privileges']:
if mysqldump.version < (5,0,26):
LOG.warning("--flush privileges is available only for mysqldump "
"in 5.0.26+")
else:
options.append('--flush-privileges')
if config['dump-routines']:
if mysqldump.version < (5, 0, 13):
LOG.warning("--routines is not available before mysqldump 5.0.13+")
else:
if mysqldump.version < (5, 0, 20):
LOG.warning("mysqldump will not dump DEFINER values before "
"version 5.0.20. You are running mysqldump from "
"version %s", mysqldump.version_str)
options.append('--routines')
if config['dump-events']:
if mysqldump.version < (5, 1, 8):
LOG.warning("--events only available for mysqldump 5.1.8+. skipping")
else:
options.append('--events')
if config['max-allowed-packet']:
options.append('--max-allowed-packet=' + config['max-allowed-packet'])
if config['bin-log-position']:
if client.show_variable('log_bin') != 'ON':
raise BackupError("bin-log-position requested but "
"bin-log on server not active")
options.append('--master-data=2')
options.extend(config['additional-options'])
return options
def backup(self):
"""Run a MySQL backup"""
if self.schema.timestamp is None:
self._fast_refresh_schema()
mock_env = None
if self.dry_run:
mock_env = MockEnvironment()
mock_env.replace_environment()
LOG.info("Running in dry-run mode.")
try:
if self.config['mysqldump']['stop-slave']:
self.client = connect(self.mysql_config['client'])
if self.client.show_status('Slave_running',
session=None) != 'ON':
raise BackupError(
"stop-slave enabled, but replication is "
"either not configured or the slave is not "
"running.")
self.config.setdefault('mysql:replication', {})
_stop_slave(self.client, self.config['mysql:replication'])
self._backup()
finally:
if self.config['mysqldump']['stop-slave'] and \
'mysql:replication' in self.config:
_start_slave(self.client, self.config['mysql:replication'])
if mock_env:
mock_env.restore_environment()
def _backup(self):
"""Real backup method. May raise BackupError exceptions"""
config = self.config['mysqldump']
# setup defaults_file with ignore-table exclusions
defaults_file = os.path.join(self.target_directory, 'my.cnf')
write_options(self.mysql_config, defaults_file)
if config['exclude-invalid-views']:
LOG.info("* Finding and excluding invalid views...")
definitions_path = os.path.join(self.target_directory,
'invalid_views.sql')
exclude_invalid_views(self.schema, self.client, definitions_path)
add_exclusions(self.schema, defaults_file)
# find the path to the mysqldump command
mysqldump_bin = find_mysqldump(path=config['mysql-binpath'])
LOG.info("Using mysqldump executable: %s", mysqldump_bin)
# setup the mysqldump environment
extra_defaults = config['extra-defaults']
try:
mysqldump = MySQLDump(defaults_file,
mysqldump_bin,
extra_defaults=extra_defaults)
except MySQLDumpError, exc:
raise BackupError(str(exc))
def _stop_slave(client, config=None):
"""Stop MySQL replication"""
try:
client.stop_slave(sql_thread_only=True)
LOG.info("Stopped slave")
except MySQLError, exc:
raise BackupError("Failed to stop slave[%d]: %s" % exc.args)
def __call__(self, event, snapshot_fsm, snapshot):
LOG.info("Starting InnoDB recovery")
mysqld_exe = locate_mysqld_exe(self.mysqld_config)
LOG.info("Bootstrapping with %s", mysqld_exe)
mycnf_path = os.path.join(self.mysqld_config['datadir'],
'my.innodb_recovery.cnf')
self.mysqld_config['log-error'] = 'innodb_recovery.log'
my_conf = generate_server_config(self.mysqld_config, mycnf_path)
mysqld = MySQLServer(mysqld_exe, my_conf)
mysqld.start(bootstrap=True)
while mysqld.poll() is None:
if signal.SIGINT in snapshot_fsm.sigmgr.pending:
mysqld.kill(signal.SIGKILL)
time.sleep(0.5)
LOG.info("%s has stopped", mysqld_exe)
if mysqld.returncode != 0:
datadir = self.mysqld_config['datadir']
for line in open(os.path.join(datadir, 'innodb_recovery.log'),
'r'):
LOG.error("%s", line.rstrip())
raise BackupError("%s exited with non-zero status (%s) during "
"InnoDB recovery" %
(mysqld_exe, mysqld.returncode))
else:
LOG.info("%s ran successfully", mysqld_exe)
def _start_slave(client, config=None):
"""Start MySQL replication"""
# Skip sanity check on slave coords if we didn't actually record any coords
# This might happen if mysqld goes away between STOP SLAVE and SHOW SLAVE
# STATUS.
if config:
try:
slave_info = client.show_slave_status()
if slave_info and slave_info['exec_master_log_pos'] != config['slave_master_log_pos']:
LOG.warning("Sanity check on slave status failed. "
"Previously recorded %s:%d but currently found %s:%d",
config['slave_master_log_file'], config['slave_master_log_pos'],
slave_info['relay_master_log_file'], slave_info['exec_master_log_pos'])
LOG.warning("ALERT! Slave position changed during backup!")
except MySQLError as exc:
LOG.warning("Failed to sanity check replication[%d]: %s",
*exc.args)
try:
master_info = client.show_master_status()
if master_info and master_info['position'] != config['master_log_pos']:
LOG.warning("Sanity check on master status failed. "
"Previously recorded %s:%s but currently found %s:%s",
config['master_log_file'], config['master_log_pos'],
master_info['file'], master_info['position'])
LOG.warning("ALERT! Binary log position changed during backup!")
except MySQLError as exc:
LOG.warning("Failed to sanity check master status. [%d] %s", *exc.args)
try:
client.start_slave()
LOG.info("Restarted slave")
except MySQLError as exc:
raise BackupError("Failed to restart slave [%d] %s" % exc.args)
class DelphiniPlugin(object):
"""MySQL Cluster Backup Plugin implementation for Holland"""
def __init__(self, name, config, target_directory, dry_run=False):
config.validate_config(self.configspec())
self.name = name
self.config = config
self.target_directory = target_directory
self.dry_run = dry_run
def estimate_backup_size(self):
"""Estimate the backup size"""
# XXX: implement I_S querying or ssh du -sh perhaps
return 0
def backup(self):
"""Run a MySQL cluster backup"""
config = self.config['mysql-cluster']
dsn = config['connect-string']
ssh_user = config['default-ssh-user']
ssh_keyfile = config['default-ssh-keyfile']
target_path = os.path.join(self.target_directory, 'data')
try:
os.mkdir(target_path)
except OSError, exc:
raise BackupError("Failed to create %s: %s", (target_path, exc))
try:
backup_id, stop_gcp = backup(dsn, ssh_user, ssh_keyfile,
target_path)
except ClusterError, exc:
raise BackupError(exc)
def backup(self):
"""
Start a backup. This attempts one or more mysqldump
runs. On error, a BackupError exception will be thrown.
"""
if self.config.lookup('mysqlhotcopy.stop-slave'):
self.client.stop_slave()
if self.config.lookup('mysqlhotcopy.bin-log-position'):
#ensure mysql:replication section exists
self.config.setdefault('mysql:replication', {})
# Log slave data if we can:
is_slave = self.client.is_slave_running()
if is_slave:
slave_status = self.client.show_slave_status()
self.config['mysql:replication'][
'slave_master_log_file'] = slave_status[
'Relay_Master_Log_File']
self.config['mysql:replication'][
'slave_master_log_pos'] = slave_status[
'Exec_Master_Log_Pos']
master_data = self.client.show_master_status()
if not master_data and not is_slave:
LOG.error(
"bin-log-position requested, but this server is neither a master nor a slave"
)
raise BackupError("Failboat: replication not configured")
self.config['mysql:replication']['master_log_file'] = master_data[
0]
self.config['mysql:replication']['master_log_pos'] = master_data[1]
LOG.info("Writing master status to %s", self.mysql_config.path)
if not self.dry_run:
self.config.write()
# trap exceptions so we make sure to restart the slave, if we stopped it
# if the slave was already stopped, we will raise an exception when we
# try to stop it (above)
error = None
try:
self._backup()
except Exception as ex:
raise BackupError(ex)
finally:
if self.config.lookup('mysqlhotcopy.stop-slave'):
self.client.start_slave()
def backup(backupset_name, dry_run=False, skip_purge=False):
# May raise a ConfigError if not backupset is found
LOGGER.info("Loading config for backupset %s", backupset_name)
try:
backupset_cfg = load_backupset_config(backupset_name)
except IOError, e:
LOGGER.error("Failed to load backupset %s: %s", backupset_name, e)
raise BackupError("Aborting due to previous errors.")
def _estimate_legacy_size(self, db):
try:
connection = get_connection(self.config, db)
size = legacy_get_db_size(db, connection)
connection.close()
return size
except dbapi.DatabaseError, exc:
raise BackupError("Failed to estimate database size for %s: %s" %
(db, exc))
def find_mysqldump(path=None):
"""Find a usable mysqldump binary in path or ENV[PATH]"""
search_path = ':'.join(path) or os.environ.get('PATH', '')
for _path in search_path.split(':'):
if os.path.isfile(_path):
return os.path.realpath(_path)
if os.path.exists(os.path.join(_path, 'mysqldump')):
return os.path.realpath(os.path.join(_path, 'mysqldump'))
raise BackupError("Failed to find mysqldump in %s" % search_path)
def verify_space(required_space, target_directory):
available_space = disk_free(_find_existing_parent(target_directory))
if required_space >= available_space:
LOGGER.error("Insufficient Disk Space. Required: %s Available: %s",
format_bytes(required_space),
format_bytes(available_space))
raise BackupError("%s required but only %s available on %s" % \
(format_bytes(required_space),
format_bytes(available_space),
target_directory))
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will produce"""
try:
mysql_config = build_mysql_config(self.config['mysql:client'])
client = connect(mysql_config['client'])
datadir = client.show_variable('datadir')
return directory_size(datadir)
except MySQLError, exc:
raise BackupError("Failed to lookup the MySQL datadir when "
"estimating backup size: [%d] %s" % exc.args)
def estimate_backup_size(self):
"""Estimate the size of the backup this plugin will generate"""
LOG.info("Estimating size of mysqldump backup")
estimate_method = self.config['mysqldump']['estimate-method']
if estimate_method.startswith('const:'):
try:
return parse_size(estimate_method[6:])
except ValueError, exc:
raise BackupError(str(exc))
def backup(self):
"""Run a backup by running through a LVM snapshot against the device
the MySQL datadir resides on
"""
# connect to mysql and lookup what we're supposed to snapshot
try:
self.client.connect()
datadir = os.path.realpath(self.client.show_variable('datadir'))
except MySQLError, exc:
raise BackupError("[%d] %s" % exc.args)
def estimate_backup_size(self):
"""Estimate the backup size this plugin will produce
This is currently the total directory size of the MySQL datadir
"""
try:
self.client.connect()
datadir = self.client.show_variable('datadir')
self.client.disconnect()
except MySQLError, exc:
raise BackupError("[%d] %s" % exc.args)
def __call__(self, event, snapshot_fsm, snapshot_vol):
argv = [
'tar', '--create', '--file', '-', '--verbose', '--totals',
'--directory', self.snap_datadir, '.'
]
pre_args = self.config['pre-args']
if pre_args:
LOG.info("Adding tar pre-args: %s", pre_args)
pre_args = [
arg.decode('utf8')
for arg in shlex.split(pre_args.encode('utf8'))
]
for option in pre_args:
argv.insert(-3, option)
for param in self.config['exclude']:
argv.insert(-1, "--exclude")
argv.insert(-1, os.path.join('.', param))
post_args = self.config['post-args']
if post_args:
LOG.info("Adding tar post-args: %s", post_args)
post_args = [
arg.decode('utf8')
for arg in shlex.split(post_args.encode('utf8'))
]
for option in post_args:
argv.append(option)
LOG.info("Running: %s > %s", list2cmdline(argv),
self.archive_stream.name)
archive_dirname = os.path.dirname(self.archive_stream.name)
if pre_args or post_args:
warning_readme = os.path.join(archive_dirname, "NONSTD_TAR.txt")
warning_log = open('warning_readme', 'w')
print >> warning_log, ("This tar file was generated with non-std "
"args:")
print >> warning_log, list2cmdline(argv)
archive_log = os.path.join(archive_dirname, 'archive.log')
process = Popen(argv,
preexec_fn=os.setsid,
stdout=self.archive_stream,
stderr=open(archive_log, 'w'),
close_fds=True)
while process.poll() is None:
if signal.SIGINT in snapshot_fsm.sigmgr.pending:
os.kill(process.pid, signal.SIGKILL)
time.sleep(0.5)
try:
self.archive_stream.close()
except IOError, exc:
LOG.error("tar output stream %s failed: %s",
self.archive_stream.name, exc)
raise BackupError(str(exc))
def backup(self):
"""Run a MySQL cluster backup"""
config = self.config['mysql-cluster']
dsn = config['connect-string']
ssh_user = config['default-ssh-user']
ssh_keyfile = config['default-ssh-keyfile']
target_path = os.path.join(self.target_directory, 'data')
try:
os.mkdir(target_path)
except OSError, exc:
raise BackupError("Failed to create %s: %s", (target_path, exc))
def connect_simple(config):
"""Create a MySQLClientConnection given a mysql:client config
section from a holland mysql backupset
"""
try:
mysql_config = build_mysql_config(config)
LOG.debug("mysql_config => %r", mysql_config)
connection = connect(mysql_config['client'], PassiveMySQLClient)
connection.connect()
return connection
except MySQLError, exc:
raise BackupError("[%d] %s" % exc.args)
