def test_execute_hook_on_error_logs_as_error():
flexmock(module).should_receive('interpolate_context').replace_with(
lambda command, context: command)
flexmock(module.execute).should_receive('execute_command').with_args(
[':'], output_log_level=logging.ERROR, shell=True).once()
module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
def test_execute_hook_with_dry_run_skips_commands():
flexmock(module).should_receive('interpolate_context').replace_with(
lambda command, context: command
)
flexmock(module.execute).should_receive('execute_command').never()
module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True)
def test_execute_hook_invokes_each_command():
flexmock(module).should_receive('interpolate_context').replace_with(
lambda command, context: command
)
flexmock(module.execute).should_receive('execute_command').with_args(
[':'], output_log_level=logging.WARNING, shell=True
).once()
module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False)
def test_execute_hook_with_umask_sets_that_umask():
flexmock(module).should_receive('interpolate_context').replace_with(
lambda command, context: command)
flexmock(module.os).should_receive('umask').with_args(0o77).and_return(
0o22).once()
flexmock(module.os).should_receive('umask').with_args(0o22).once()
flexmock(module.execute).should_receive('execute_command').with_args(
[':'], output_log_level=logging.WARNING, shell=True)
module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False)
def collect_configuration_run_summary_logs(configs, arguments):
'''
Given a dict of configuration filename to corresponding parsed configuration, and parsed
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
each configuration file and yield a series of logging.LogRecord instances containing summary
information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Run cross-file validation checks.
if 'extract' in arguments:
repository = arguments['extract'].repository
elif 'list' in arguments and arguments['list'].archive:
repository = arguments['list'].repository
else:
repository = None
if repository:
try:
validate.guard_configuration_contains_repository(
repository, configs)
except ValueError as error:
yield from make_error_log_records(str(error))
return
if not configs:
yield from make_error_log_records(
'{}: No configuration files found'.format(' '.join(
arguments['global'].config_paths)))
return
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('before_everything'),
hooks.get('umask'),
config_filename,
'pre-everything',
arguments['global'].dry_run,
)
except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(
'Error running pre-everything hook', error)
return
# Execute the actions corresponding to each configuration file.
json_results = []
for config_filename, config in configs.items():
results = list(run_configuration(config_filename, config, arguments))
error_logs = tuple(result for result in results
if isinstance(result, logging.LogRecord))
if error_logs:
yield from make_error_log_records(
'{}: Error running configuration file'.format(config_filename))
yield from error_logs
else:
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
levelname='INFO',
msg='{}: Successfully ran configuration file'.format(
config_filename),
))
if results:
json_results.extend(results)
if json_results:
sys.stdout.write(json.dumps(json_results))
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('after_everything'),
hooks.get('umask'),
config_filename,
'post-everything',
arguments['global'].dry_run,
)
except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(
'Error running post-everything hook', error)
def run_configuration(config_filename, config, arguments):
'''
Given a config filename, the corresponding parsed config dict, and command-line arguments as a
dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
backups, consistency checks, and/or other actions.
Yield a combination of:
* JSON output strings from successfully executing any actions that produce JSON
* logging.LogRecord instances containing errors from any actions or backup hooks that fail
'''
(location, storage, retention, consistency,
hooks) = (config.get(section_name, {})
for section_name in ('location', 'storage', 'retention',
'consistency', 'hooks'))
global_arguments = arguments['global']
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
borg_environment.initialize(storage)
encountered_error = None
error_repository = ''
if 'create' in arguments:
try:
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run, 'start')
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
)
postgresql.dump_databases(hooks.get('postgresql_databases'),
config_filename,
global_arguments.dry_run)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
'{}: Error running pre-backup hook'.format(config_filename),
error)
if not encountered_error:
for repository_path in location['repositories']:
try:
yield from run_actions(
arguments=arguments,
location=location,
storage=storage,
retention=retention,
consistency=consistency,
local_path=local_path,
remote_path=remote_path,
repository_path=repository_path,
)
except (OSError, CalledProcessError) as error:
encountered_error = error
error_repository = repository_path
yield from make_error_log_records(
'{}: Error running actions for repository'.format(
repository_path), error)
if 'create' in arguments and not encountered_error:
try:
postgresql.remove_database_dumps(hooks.get('postgresql_databases'),
config_filename,
global_arguments.dry_run)
command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,
)
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
'{}: Error running post-backup hook'.format(config_filename),
error)
if encountered_error:
try:
command.execute_hook(
hooks.get('on_error'),
hooks.get('umask'),
config_filename,
'on-error',
global_arguments.dry_run,
repository=error_repository,
error=encountered_error,
output=getattr(encountered_error, 'output', ''),
)
healthchecks.ping_healthchecks(hooks.get('healthchecks'),
config_filename,
global_arguments.dry_run, 'fail')
except (OSError, CalledProcessError) as error:
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename),
error)
def run_configuration(config_filename, config, arguments):
'''
Given a config filename, the corresponding parsed config dict, and command-line arguments as a
dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
backups, consistency checks, and/or other actions.
Yield a combination of:
* JSON output strings from successfully executing any actions that produce JSON
* logging.LogRecord instances containing errors from any actions or backup hooks that fail
'''
(location, storage, retention, consistency,
hooks) = (config.get(section_name, {})
for section_name in ('location', 'storage', 'retention',
'consistency', 'hooks'))
global_arguments = arguments['global']
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
borg_environment.initialize(storage)
encountered_error = None
error_repository = ''
prune_create_or_check = {'prune', 'create',
'check'}.intersection(arguments)
monitoring_log_level = verbosity_to_log_level(
global_arguments.monitoring_verbosity)
try:
if prune_create_or_check:
dispatch.call_hooks(
'ping_monitor',
hooks,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.START,
monitoring_log_level,
global_arguments.dry_run,
)
if 'prune' in arguments:
command.execute_hook(
hooks.get('before_prune'),
hooks.get('umask'),
config_filename,
'pre-prune',
global_arguments.dry_run,
)
if 'create' in arguments:
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
)
dispatch.call_hooks(
'dump_databases',
hooks,
config_filename,
dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
if 'check' in arguments:
command.execute_hook(
hooks.get('before_check'),
hooks.get('umask'),
config_filename,
'pre-check',
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
encountered_error = error
yield from make_error_log_records(
'{}: Error running pre hook'.format(config_filename), error)
if not encountered_error:
for repository_path in location['repositories']:
try:
yield from run_actions(
arguments=arguments,
location=location,
storage=storage,
retention=retention,
consistency=consistency,
hooks=hooks,
local_path=local_path,
remote_path=remote_path,
repository_path=repository_path,
)
except (OSError, CalledProcessError, ValueError) as error:
encountered_error = error
error_repository = repository_path
yield from make_error_log_records(
'{}: Error running actions for repository'.format(
repository_path), error)
if not encountered_error:
try:
if 'prune' in arguments:
command.execute_hook(
hooks.get('after_prune'),
hooks.get('umask'),
config_filename,
'post-prune',
global_arguments.dry_run,
)
if 'create' in arguments:
dispatch.call_hooks(
'remove_database_dumps',
hooks,
config_filename,
dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,
)
if 'check' in arguments:
command.execute_hook(
hooks.get('after_check'),
hooks.get('umask'),
config_filename,
'post-check',
global_arguments.dry_run,
)
if {'prune', 'create', 'check'}.intersection(arguments):
dispatch.call_hooks(
'ping_monitor',
hooks,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.FINISH,
monitoring_log_level,
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
encountered_error = error
yield from make_error_log_records(
'{}: Error running post hook'.format(config_filename), error)
if encountered_error and prune_create_or_check:
try:
command.execute_hook(
hooks.get('on_error'),
hooks.get('umask'),
config_filename,
'on-error',
global_arguments.dry_run,
repository=error_repository,
error=encountered_error,
output=getattr(encountered_error, 'output', ''),
)
dispatch.call_hooks(
'ping_monitor',
hooks,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.FAIL,
monitoring_log_level,
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename),
error)
def test_execute_hook_with_empty_commands_does_not_raise():
module.execute_hook([], None, 'config.yaml', 'post-backup', dry_run=False)
