def test_make_database_dump_filename_uses_name_and_hostname():
flexmock(module.os.path).should_receive('expanduser').and_return('databases')
assert (
module.make_database_dump_filename('databases', 'test', 'hostname')
== 'databases/hostname/test'
)
def restore_database_dumps(databases, log_prefix, location_config, dry_run):
'''
Restore the given MySQL/MariaDB databases from disk. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path. If this is a dry run, then don't actually restore anything.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
for database in databases:
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
)
restore_command = (
('mysql', '--batch')
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', database['username']) if 'username' in database else ())
)
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
logger.debug(
'{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
)
if not dry_run:
execute_command(
restore_command, input_file=open(dump_filename), extra_environment=extra_environment
)
def restore_database_dump(database_config, log_prefix, location_config,
dry_run, extract_process):
'''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database = database_config[0]
all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'],
database.get('hostname'))
analyze_command = (
('psql', '--no-password', '--quiet') +
(('--host', database['hostname']) if 'hostname' in database else
()) + (('--port', str(database['port'])) if 'port' in database else
()) +
(('--username', database['username']) if 'username' in database else
()) + (('--dbname', database['name']) if not all_databases else
()) + ('--command', 'ANALYZE'))
restore_command = (
('psql' if all_databases else 'pg_restore', '--no-password') +
(('--if-exists', '--exit-on-error', '--clean', '--dbname',
database['name']) if not all_databases else ()) +
(('--host', database['hostname']) if 'hostname' in database else
()) + (('--port', str(database['port'])) if 'port' in database else
()) +
(('--username', database['username']) if 'username' in database else
()) + (() if extract_process else (dump_filename, )))
extra_environment = {
'PGPASSWORD': database['password']
} if 'password' in database else None
logger.debug('{}: Restoring PostgreSQL database {}{}'.format(
log_prefix, database['name'], dry_run_label))
if dry_run:
return
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
extra_environment=extra_environment,
borg_local_path=location_config.get('local_path', 'borg'),
)
execute_command(analyze_command, extra_environment=extra_environment)
def make_database_dump_patterns(databases, log_prefix, location_config, names):
'''
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
and a sequence of database names to match, return the corresponding glob patterns to match the
database dumps in an archive. An empty sequence of names indicates that the patterns should
match all dumps.
'''
return [
dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
for name in (names or ['*'])
]
def make_database_dump_pattern(databases,
log_prefix,
location_config,
name=None): # pragma: no cover
'''
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
and a database name to match, return the corresponding glob patterns to match the database dump
in an archive.
'''
return dump.make_database_dump_filename(make_dump_path(location_config),
name,
hostname='*')
def dump_databases(databases, log_prefix, location_config, dry_run):
'''
Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path. If this is a dry run, then don't actually dump anything.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
for database in databases:
name = database['name']
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), name, database.get('hostname')
)
command = (
('mysqldump', '--add-drop-database')
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', database['username']) if 'username' in database else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
+ (('--all-databases',) if name == 'all' else ('--databases', name))
)
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
logger.debug(
'{}: Dumping MySQL database {} to {}{}'.format(
log_prefix, name, dump_filename, dry_run_label
)
)
if not dry_run:
os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
execute_command(
command, output_file=open(dump_filename, 'w'), extra_environment=extra_environment
)
def test_make_database_dump_filename_with_invalid_name_raises():
flexmock(module.os.path).should_receive('expanduser').and_return('databases')
with pytest.raises(ValueError):
module.make_database_dump_filename('databases', 'invalid/name')
def test_make_database_dump_filename_without_hostname_defaults_to_localhost():
flexmock(module.os.path).should_receive('expanduser').and_return('databases')
assert module.make_database_dump_filename('databases', 'test') == 'databases/localhost/test'
def dump_databases(databases, log_prefix, location_config, dry_run):
'''
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
processes = []
logger.info('{}: Dumping PostgreSQL databases{}'.format(
log_prefix, dry_run_label))
for database in databases:
name = database['name']
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), name, database.get('hostname'))
all_databases = bool(name == 'all')
dump_format = database.get('format', 'custom')
command = (
(
'pg_dumpall' if all_databases else 'pg_dump',
'--no-password',
'--clean',
'--if-exists',
) +
(('--host', database['hostname']) if 'hostname' in database else
()) +
(('--port', str(database['port'])) if 'port' in database else
()) + (('--username',
database['username']) if 'username' in database else
()) + (() if all_databases else
('--format', dump_format)) +
(('--file', dump_filename) if dump_format == 'directory' else
()) + (tuple(database['options'].split(' ')) if 'options'
in database else ()) + (() if all_databases else (name, ))
# Use shell redirection rather than the --file flag to sidestep synchronization issues
# when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
# format in a particular, a named destination is required, and redirection doesn't work.
+ (('>', dump_filename) if dump_format != 'directory' else ()))
extra_environment = {
'PGPASSWORD': database['password']
} if 'password' in database else None
logger.debug('{}: Dumping PostgreSQL database {} to {}{}'.format(
log_prefix, name, dump_filename, dry_run_label))
if dry_run:
continue
if dump_format == 'directory':
dump.create_parent_directory_for_dump(dump_filename)
else:
dump.create_named_pipe_for_dump(dump_filename)
processes.append(
execute_command(command,
shell=True,
extra_environment=extra_environment,
run_to_completion=False))
return processes
def dump_databases(databases, log_prefix, location_config, dry_run):
'''
Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
of dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
processes = []
logger.info('{}: Dumping MySQL databases{}'.format(log_prefix,
dry_run_label))
for database in databases:
requested_name = database['name']
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), requested_name,
database.get('hostname'))
extra_environment = {
'MYSQL_PWD': database['password']
} if 'password' in database else None
dump_database_names = database_names_to_dump(database,
extra_environment,
log_prefix, dry_run_label)
if not dump_database_names:
raise ValueError('Cannot find any MySQL databases to dump.')
dump_command = (
('mysqldump', ) + ('--add-drop-database', ) +
(('--host', database['hostname']) if 'hostname' in database else
()) +
(('--port', str(database['port'])) if 'port' in database else ()) +
(('--protocol',
'tcp') if 'hostname' in database or 'port' in database else ()) +
(('--user', database['username']) if 'username' in database else
()) + (tuple(database['options'].split(' '))
if 'options' in database else
()) + ('--databases', ) + dump_database_names
# Use shell redirection rather than execute_command(output_file=open(...)) to prevent
# the open() call on a named pipe from hanging the main borgmatic process.
+ ('>', dump_filename))
logger.debug('{}: Dumping MySQL database {} to {}{}'.format(
log_prefix, requested_name, dump_filename, dry_run_label))
if dry_run:
continue
dump.create_named_pipe_for_dump(dump_filename)
processes.append(
execute_command(
dump_command,
shell=True,
extra_environment=extra_environment,
run_to_completion=False,
))
return processes
