def test_execute_hook_on_error_logs_as_error(): flexmock(module).should_receive('interpolate_context').replace_with( lambda command, context: command) flexmock(module.execute).should_receive('execute_command').with_args( [':'], output_log_level=logging.ERROR, shell=True).once() module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
def test_execute_hook_with_dry_run_skips_commands(): flexmock(module).should_receive('interpolate_context').replace_with( lambda command, context: command ) flexmock(module.execute).should_receive('execute_command').never() module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True)
def test_execute_hook_invokes_each_command(): flexmock(module).should_receive('interpolate_context').replace_with( lambda command, context: command ) flexmock(module.execute).should_receive('execute_command').with_args( [':'], output_log_level=logging.WARNING, shell=True ).once() module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False)
def test_execute_hook_with_umask_sets_that_umask(): flexmock(module).should_receive('interpolate_context').replace_with( lambda command, context: command) flexmock(module.os).should_receive('umask').with_args(0o77).and_return( 0o22).once() flexmock(module.os).should_receive('umask').with_args(0o22).once() flexmock(module.execute).should_receive('execute_command').with_args( [':'], output_log_level=logging.WARNING, shell=True) module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False)
def collect_configuration_run_summary_logs(configs, arguments): ''' Given a dict of configuration filename to corresponding parsed configuration, and parsed command-line arguments as a dict from subparser name to a parsed namespace of arguments, run each configuration file and yield a series of logging.LogRecord instances containing summary information about each run. As a side effect of running through these configuration files, output their JSON results, if any, to stdout. ''' # Run cross-file validation checks. if 'extract' in arguments: repository = arguments['extract'].repository elif 'list' in arguments and arguments['list'].archive: repository = arguments['list'].repository else: repository = None if repository: try: validate.guard_configuration_contains_repository( repository, configs) except ValueError as error: yield from make_error_log_records(str(error)) return if not configs: yield from make_error_log_records( '{}: No configuration files found'.format(' '.join( arguments['global'].config_paths))) return if 'create' in arguments: try: for config_filename, config in configs.items(): hooks = config.get('hooks', {}) command.execute_hook( hooks.get('before_everything'), hooks.get('umask'), config_filename, 'pre-everything', arguments['global'].dry_run, ) except (CalledProcessError, ValueError, OSError) as error: yield from make_error_log_records( 'Error running pre-everything hook', error) return # Execute the actions corresponding to each configuration file. json_results = [] for config_filename, config in configs.items(): results = list(run_configuration(config_filename, config, arguments)) error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: yield from make_error_log_records( '{}: Error running configuration file'.format(config_filename)) yield from error_logs else: yield logging.makeLogRecord( dict( levelno=logging.INFO, levelname='INFO', msg='{}: Successfully ran configuration file'.format( config_filename), )) if results: json_results.extend(results) if json_results: sys.stdout.write(json.dumps(json_results)) if 'create' in arguments: try: for config_filename, config in configs.items(): hooks = config.get('hooks', {}) command.execute_hook( hooks.get('after_everything'), hooks.get('umask'), config_filename, 'post-everything', arguments['global'].dry_run, ) except (CalledProcessError, ValueError, OSError) as error: yield from make_error_log_records( 'Error running post-everything hook', error)
def run_configuration(config_filename, config, arguments): ''' Given a config filename, the corresponding parsed config dict, and command-line arguments as a dict from subparser name to a namespace of parsed arguments, execute its defined pruning, backups, consistency checks, and/or other actions. Yield a combination of: * JSON output strings from successfully executing any actions that produce JSON * logging.LogRecord instances containing errors from any actions or backup hooks that fail ''' (location, storage, retention, consistency, hooks) = (config.get(section_name, {}) for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')) global_arguments = arguments['global'] local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') borg_environment.initialize(storage) encountered_error = None error_repository = '' if 'create' in arguments: try: healthchecks.ping_healthchecks(hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start') command.execute_hook( hooks.get('before_backup'), hooks.get('umask'), config_filename, 'pre-backup', global_arguments.dry_run, ) postgresql.dump_databases(hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run) except (OSError, CalledProcessError) as error: encountered_error = error yield from make_error_log_records( '{}: Error running pre-backup hook'.format(config_filename), error) if not encountered_error: for repository_path in location['repositories']: try: yield from run_actions( arguments=arguments, location=location, storage=storage, retention=retention, consistency=consistency, local_path=local_path, remote_path=remote_path, repository_path=repository_path, ) except (OSError, CalledProcessError) as error: encountered_error = error error_repository = repository_path yield from make_error_log_records( '{}: Error running actions for repository'.format( repository_path), error) if 'create' in arguments and not encountered_error: try: postgresql.remove_database_dumps(hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run) command.execute_hook( hooks.get('after_backup'), hooks.get('umask'), config_filename, 'post-backup', global_arguments.dry_run, ) healthchecks.ping_healthchecks(hooks.get('healthchecks'), config_filename, global_arguments.dry_run) except (OSError, CalledProcessError) as error: encountered_error = error yield from make_error_log_records( '{}: Error running post-backup hook'.format(config_filename), error) if encountered_error: try: command.execute_hook( hooks.get('on_error'), hooks.get('umask'), config_filename, 'on-error', global_arguments.dry_run, repository=error_repository, error=encountered_error, output=getattr(encountered_error, 'output', ''), ) healthchecks.ping_healthchecks(hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail') except (OSError, CalledProcessError) as error: yield from make_error_log_records( '{}: Error running on-error hook'.format(config_filename), error)
def run_configuration(config_filename, config, arguments): ''' Given a config filename, the corresponding parsed config dict, and command-line arguments as a dict from subparser name to a namespace of parsed arguments, execute its defined pruning, backups, consistency checks, and/or other actions. Yield a combination of: * JSON output strings from successfully executing any actions that produce JSON * logging.LogRecord instances containing errors from any actions or backup hooks that fail ''' (location, storage, retention, consistency, hooks) = (config.get(section_name, {}) for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')) global_arguments = arguments['global'] local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') borg_environment.initialize(storage) encountered_error = None error_repository = '' prune_create_or_check = {'prune', 'create', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level( global_arguments.monitoring_verbosity) try: if prune_create_or_check: dispatch.call_hooks( 'ping_monitor', hooks, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.START, monitoring_log_level, global_arguments.dry_run, ) if 'prune' in arguments: command.execute_hook( hooks.get('before_prune'), hooks.get('umask'), config_filename, 'pre-prune', global_arguments.dry_run, ) if 'create' in arguments: command.execute_hook( hooks.get('before_backup'), hooks.get('umask'), config_filename, 'pre-backup', global_arguments.dry_run, ) dispatch.call_hooks( 'dump_databases', hooks, config_filename, dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, ) if 'check' in arguments: command.execute_hook( hooks.get('before_check'), hooks.get('umask'), config_filename, 'pre-check', global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return encountered_error = error yield from make_error_log_records( '{}: Error running pre hook'.format(config_filename), error) if not encountered_error: for repository_path in location['repositories']: try: yield from run_actions( arguments=arguments, location=location, storage=storage, retention=retention, consistency=consistency, hooks=hooks, local_path=local_path, remote_path=remote_path, repository_path=repository_path, ) except (OSError, CalledProcessError, ValueError) as error: encountered_error = error error_repository = repository_path yield from make_error_log_records( '{}: Error running actions for repository'.format( repository_path), error) if not encountered_error: try: if 'prune' in arguments: command.execute_hook( hooks.get('after_prune'), hooks.get('umask'), config_filename, 'post-prune', global_arguments.dry_run, ) if 'create' in arguments: dispatch.call_hooks( 'remove_database_dumps', hooks, config_filename, dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, ) command.execute_hook( hooks.get('after_backup'), hooks.get('umask'), config_filename, 'post-backup', global_arguments.dry_run, ) if 'check' in arguments: command.execute_hook( hooks.get('after_check'), hooks.get('umask'), config_filename, 'post-check', global_arguments.dry_run, ) if {'prune', 'create', 'check'}.intersection(arguments): dispatch.call_hooks( 'ping_monitor', hooks, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.FINISH, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return encountered_error = error yield from make_error_log_records( '{}: Error running post hook'.format(config_filename), error) if encountered_error and prune_create_or_check: try: command.execute_hook( hooks.get('on_error'), hooks.get('umask'), config_filename, 'on-error', global_arguments.dry_run, repository=error_repository, error=encountered_error, output=getattr(encountered_error, 'output', ''), ) dispatch.call_hooks( 'ping_monitor', hooks, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.FAIL, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return yield from make_error_log_records( '{}: Error running on-error hook'.format(config_filename), error)
def test_execute_hook_with_empty_commands_does_not_raise(): module.execute_hook([], None, 'config.yaml', 'post-backup', dry_run=False)