def test_dump_databases_runs_pg_dump_with_hostname_and_port():
databases = [{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433
}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(
module.dump).should_receive('make_database_dump_filename').and_return(
'databases/database.example.org/foo')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--file',
'databases/database.example.org/foo',
'--host',
'database.example.org',
'--port',
'5433',
'--format',
'custom',
'foo',
),
extra_environment=None,
).once()
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
def test_dump_databases_runs_pg_dump_with_username_and_password():
databases = [{
'name': 'foo',
'username': '******',
'password': '******'
}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive(
'make_database_dump_filename').and_return('databases/localhost/foo')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--file',
'databases/localhost/foo',
'--username',
'postgres',
'--format',
'custom',
'foo',
),
extra_environment={
'PGPASSWORD': '******'
},
).once()
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
def test_dump_databases_with_dry_run_skips_pg_dump():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(
module.os.path).should_receive('expanduser').and_return('databases')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').never()
module.dump_databases(databases, 'test.yaml', dry_run=True)
def test_dump_databases_with_dry_run_skips_pg_dump():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(
module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/foo').and_return('databases/localhost/bar')
flexmock(module.os).should_receive('makedirs').never()
flexmock(module).should_receive('execute_command').never()
module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
def test_dump_databases_runs_pg_dumpall_for_all_databases():
databases = [{'name': 'all'}]
flexmock(
module.os.path).should_receive('expanduser').and_return('databases')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args(
('pg_dumpall', '--no-password', '--clean', '--file',
'databases/localhost/all'),
extra_environment=None,
).once()
module.dump_databases(databases, 'test.yaml', dry_run=False)
def test_dump_databases_runs_pg_dump_with_directory_format():
databases = [{'name': 'foo', 'format': 'directory'}]
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive(
'make_database_dump_filename').and_return('databases/localhost/foo')
flexmock(module.dump).should_receive('create_parent_directory_for_dump')
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--if-exists',
'--format',
'directory',
'--file',
'databases/localhost/foo',
'foo',
),
shell=True,
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=False) == [process]
def test_dump_databases_runs_pg_dump_with_username_and_password():
databases = [{
'name': 'foo',
'username': '******',
'password': '******'
}]
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive(
'make_database_dump_filename').and_return('databases/localhost/foo')
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--if-exists',
'--username',
'postgres',
'--format',
'custom',
'foo',
'>',
'databases/localhost/foo',
),
shell=True,
extra_environment={
'PGPASSWORD': '******'
},
run_to_completion=False,
).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=False) == [process]
def test_dump_databases_runs_pg_dump_for_each_database():
databases = [{'name': 'foo'}, {'name': 'bar'}]
processes = [flexmock(), flexmock()]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(
module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/foo').and_return('databases/localhost/bar')
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGSSLMODE': 'disable'})
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--if-exists',
'--format',
'custom',
name,
'>',
'databases/localhost/{}'.format(name),
),
shell=True,
extra_environment={
'PGSSLMODE': 'disable'
},
run_to_completion=False,
).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=False) == processes
def test_dump_databases_runs_pg_dump_with_options():
databases = [{'name': 'foo', 'options': '--stuff=such'}]
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive(
'make_database_dump_filename').and_return('databases/localhost/foo')
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGSSLMODE': 'disable'})
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--if-exists',
'--format',
'custom',
'--stuff=such',
'foo',
'>',
'databases/localhost/foo',
),
shell=True,
extra_environment={
'PGSSLMODE': 'disable'
},
run_to_completion=False,
).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=False) == [process]
def test_dump_databases_runs_pg_dump_with_options():
databases = [{'name': 'foo', 'options': '--stuff=such'}]
flexmock(
module.os.path).should_receive('expanduser').and_return('databases')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--file',
'databases/localhost/foo',
'--format',
'custom',
'--stuff=such',
'foo',
),
extra_environment=None,
).once()
module.dump_databases(databases, 'test.yaml', dry_run=False)
def test_dump_databases_with_dry_run_skips_pg_dump():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(
module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/foo').and_return('databases/localhost/bar')
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGSSLMODE': 'disable'})
flexmock(module).should_receive('execute_command').never()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=True) == []
def test_dump_databases_runs_pg_dump_for_each_database():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(
module.os.path).should_receive('expanduser').and_return('databases')
flexmock(module.os).should_receive('makedirs')
for name in ('foo', 'bar'):
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--file',
'databases/localhost/{}'.format(name),
'--format',
'custom',
name,
),
extra_environment=None,
).once()
module.dump_databases(databases, 'test.yaml', dry_run=False)
def test_dump_databases_runs_pg_dump_with_format():
databases = [{'name': 'foo', 'format': 'tar'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive(
'make_database_dump_filename').and_return('databases/localhost/foo')
flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--file',
'databases/localhost/foo',
'--format',
'tar',
'foo',
),
extra_environment=None,
).once()
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
def test_dump_databases_runs_pg_dump_with_hostname_and_port():
databases = [{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433
}]
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(
module.dump).should_receive('make_database_dump_filename').and_return(
'databases/database.example.org/foo')
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'pg_dump',
'--no-password',
'--clean',
'--if-exists',
'--host',
'database.example.org',
'--port',
'5433',
'--format',
'custom',
'foo',
'>',
'databases/database.example.org/foo',
),
shell=True,
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {},
dry_run=False) == [process]
def run_configuration(config_filename, config, arguments):
'''
Given a config filename, the corresponding parsed config dict, and command-line arguments as a
dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
backups, consistency checks, and/or other actions.
Yield a combination of:
* JSON output strings from successfully executing any actions that produce JSON
* logging.LogRecord instances containing errors from any actions or backup hooks that fail
'''
(location, storage, retention, consistency,
hooks) = (config.get(section_name, {})
for section_name in ('location', 'storage', 'retention',
'consistency', 'hooks'))
global_arguments = arguments['global']
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
borg_environment.initialize(storage)
encountered_error = None
error_repository = ''
if 'create' in arguments:
try:
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run, 'start')
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
)
postgresql.dump_databases(hooks.get('postgresql_databases'),
config_filename,
global_arguments.dry_run)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
'{}: Error running pre-backup hook'.format(config_filename),
error)
if not encountered_error:
for repository_path in location['repositories']:
try:
yield from run_actions(
arguments=arguments,
location=location,
storage=storage,
retention=retention,
consistency=consistency,
local_path=local_path,
remote_path=remote_path,
repository_path=repository_path,
)
except (OSError, CalledProcessError) as error:
encountered_error = error
error_repository = repository_path
yield from make_error_log_records(
'{}: Error running actions for repository'.format(
repository_path), error)
if 'create' in arguments and not encountered_error:
try:
postgresql.remove_database_dumps(hooks.get('postgresql_databases'),
config_filename,
global_arguments.dry_run)
command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,
)
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
'{}: Error running post-backup hook'.format(config_filename),
error)
if encountered_error:
try:
command.execute_hook(
hooks.get('on_error'),
hooks.get('umask'),
config_filename,
'on-error',
global_arguments.dry_run,
repository=error_repository,
error=encountered_error,
output=getattr(encountered_error, 'output', ''),
)
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run, 'fail')
except (OSError, CalledProcessError) as error:
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename),
error)
def test_dump_databases_without_databases_does_not_raise():
module.dump_databases([], 'test.yaml', dry_run=False)
def test_dump_databases_with_invalid_database_name_raises():
databases = [{'name': 'heehee/../../etc/passwd'}]
with pytest.raises(ValueError):
module.dump_databases(databases, 'test.yaml', dry_run=True)
