def restore_database_dump(database_config, log_prefix, location_config,
dry_run, extract_process):
'''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database = database_config[0]
all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'],
database.get('hostname'))
analyze_command = (
('psql', '--no-password', '--quiet') +
(('--host', database['hostname']) if 'hostname' in database else
()) + (('--port', str(database['port'])) if 'port' in database else
()) +
(('--username', database['username']) if 'username' in database else
()) + (('--dbname', database['name']) if not all_databases else
()) + ('--command', 'ANALYZE'))
restore_command = (
('psql' if all_databases else 'pg_restore', '--no-password') +
(('--if-exists', '--exit-on-error', '--clean', '--dbname',
database['name']) if not all_databases else ()) +
(('--host', database['hostname']) if 'hostname' in database else
()) + (('--port', str(database['port'])) if 'port' in database else
()) +
(('--username', database['username']) if 'username' in database else
()) + (() if extract_process else (dump_filename, )))
extra_environment = {
'PGPASSWORD': database['password']
} if 'password' in database else None
logger.debug('{}: Restoring PostgreSQL database {}{}'.format(
log_prefix, database['name'], dry_run_label))
if dry_run:
return
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
extra_environment=extra_environment,
borg_local_path=location_config.get('local_path', 'borg'),
)
execute_command(analyze_command, extra_environment=extra_environment)
def test_execute_command_with_processes_kills_processes_on_error():
full_command = ['foo', 'bar']
process = flexmock(stdout=flexmock(read=lambda count: None))
process.should_receive('poll')
process.should_receive('kill').once()
processes = (process,)
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('Popen').with_args(
full_command,
stdin=None,
stdout=module.subprocess.PIPE,
stderr=module.subprocess.STDOUT,
shell=False,
env=None,
cwd=None,
).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once()
flexmock(module).should_receive('log_outputs').never()
with pytest.raises(subprocess.CalledProcessError):
module.execute_command_with_processes(full_command, processes)
def restore_database_dump(database_config, log_prefix, location_config,
dry_run, extract_process):
'''
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database = database_config[0]
restore_command = (
('mysql', '--batch', '--verbose') +
(('--host', database['hostname']) if 'hostname' in database else
()) + (('--port', str(database['port'])) if 'port' in database else
()) +
(('--protocol',
'tcp') if 'hostname' in database or 'port' in database else
()) + (('--user', database['username']) if 'username' in database else
()))
extra_environment = {
'MYSQL_PWD': database['password']
} if 'password' in database else None
logger.debug('{}: Restoring MySQL database {}{}'.format(
log_prefix, database['name'], dry_run_label))
if dry_run:
return
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=extra_environment,
borg_local_path=location_config.get('local_path', 'borg'),
)
def test_execute_command_with_processes_calls_full_command_without_capturing_output():
full_command = ['foo', 'bar']
processes = (flexmock(),)
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('Popen').with_args(
full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None
).and_return(flexmock(wait=lambda: 0)).once()
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
flexmock(module).should_receive('log_outputs')
output = module.execute_command_with_processes(
full_command, processes, output_file=module.DO_NOT_CAPTURE
)
assert output is None
def test_execute_command_with_processes_calls_full_command_with_shell():
full_command = ['foo', 'bar']
processes = (flexmock(),)
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('Popen').with_args(
' '.join(full_command),
stdin=None,
stdout=module.subprocess.PIPE,
stderr=module.subprocess.STDOUT,
shell=True,
env=None,
cwd=None,
).and_return(flexmock(stdout=None)).once()
flexmock(module).should_receive('log_outputs')
output = module.execute_command_with_processes(full_command, processes, shell=True)
assert output is None
def test_execute_command_with_processes_calls_full_command_with_output_file():
full_command = ['foo', 'bar']
processes = (flexmock(),)
output_file = flexmock(name='test')
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('Popen').with_args(
full_command,
stdin=None,
stdout=output_file,
stderr=module.subprocess.PIPE,
shell=False,
env=None,
cwd=None,
).and_return(flexmock(stderr=None)).once()
flexmock(module).should_receive('log_outputs')
output = module.execute_command_with_processes(full_command, processes, output_file=output_file)
assert output is None
def create_archive(
dry_run,
repository,
location_config,
storage_config,
local_path='borg',
remote_path=None,
progress=False,
stats=False,
json=False,
files=False,
stream_processes=None,
):
'''
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
storage config dict, create a Borg archive and return Borg's JSON output (if any).
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
create command while also triggering the given processes to produce output.
'''
sources = deduplicate_directories(
map_directories_to_devices(
_expand_directories(
location_config['source_directories'] +
borgmatic_source_directories(
location_config.get('borgmatic_source_directory')))))
pattern_file = _write_pattern_file(location_config.get('patterns'))
exclude_file = _write_pattern_file(
_expand_home_directories(location_config.get('exclude_patterns')))
checkpoint_interval = storage_config.get('checkpoint_interval', None)
chunker_params = storage_config.get('chunker_params', None)
compression = storage_config.get('compression', None)
remote_rate_limit = storage_config.get('remote_rate_limit', None)
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
files_cache = location_config.get('files_cache')
archive_name_format = storage_config.get('archive_name_format',
DEFAULT_ARCHIVE_NAME_FORMAT)
extra_borg_options = storage_config.get('extra_borg_options',
{}).get('create', '')
full_command = (
(local_path, 'create') + _make_pattern_flags(
location_config, pattern_file.name if pattern_file else None) +
_make_exclude_flags(location_config,
exclude_file.name if exclude_file else None) +
(('--checkpoint-interval',
str(checkpoint_interval)) if checkpoint_interval else
()) + (('--chunker-params', chunker_params) if chunker_params else
()) + (('--compression', compression) if compression else ()) +
(('--remote-ratelimit',
str(remote_rate_limit)) if remote_rate_limit else
()) + (('--one-file-system', ) if
location_config.get('one_file_system') or stream_processes else
()) +
(('--numeric-owner', ) if location_config.get('numeric_owner') else
()) + (('--noatime', ) if location_config.get('atime') is False else
()) +
(('--noctime', ) if location_config.get('ctime') is False else ()) +
(('--nobirthtime', ) if location_config.get('birthtime') is False else
()) + (('--read-special', ) if
(location_config.get('read_special') or stream_processes) else
()) +
(('--nobsdflags', ) if location_config.get('bsd_flags') is False else
()) + (('--files-cache', files_cache) if files_cache else
()) + (('--remote-path', remote_path) if remote_path else
()) + (('--umask', str(umask)) if umask else ()) +
(('--lock-wait', str(lock_wait)) if lock_wait else
()) + (('--list', '--filter',
'AME-') if files and not json and not progress else ()) +
(('--info', )
if logger.getEffectiveLevel() == logging.INFO and not json else
()) + (('--stats', ) if stats and not json and not dry_run else ()) +
(('--debug',
'--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else
()) + (('--dry-run', ) if dry_run else
()) + (('--progress', ) if progress else ()) +
(('--json', ) if json else
()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else
()) + ('{repository}::{archive_name_format}'.format(
repository=repository,
archive_name_format=archive_name_format), ) + sources)
if json:
output_log_level = None
elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
output_log_level = logging.WARNING
else:
output_log_level = logging.INFO
# The progress output isn't compatible with captured and logged output, as progress messes with
# the terminal directly.
output_file = DO_NOT_CAPTURE if progress else None
if stream_processes:
return execute_command_with_processes(
full_command,
stream_processes,
output_log_level,
output_file,
borg_local_path=local_path,
)
return execute_command(full_command,
output_log_level,
output_file,
borg_local_path=local_path)