def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
schema = yaml.round_trip_load(open(validate.schema_filename()).read())
source_config = legacy.parse_configuration(
args.source_config_filename, legacy.CONFIG_FORMAT
)
source_config_file_mode = os.stat(args.source_config_filename).st_mode
source_excludes = (
open(args.source_excludes_filename).read().splitlines()
if args.source_excludes_filename
else []
)
destination_config = convert.convert_legacy_parsed_config(
source_config, source_excludes, schema
)
generate.write_configuration(
args.destination_config_filename, destination_config, mode=source_config_file_mode
)
display_result(args)
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
schema = yaml.round_trip_load(open(validate.schema_filename()).read())
source_config = legacy.parse_configuration(
args.source_config_filename, legacy.CONFIG_FORMAT
)
source_config_file_mode = os.stat(args.source_config_filename).st_mode
source_excludes = (
open(args.source_excludes_filename).read().splitlines()
if args.source_excludes_filename
else []
)
destination_config = convert.convert_legacy_parsed_config(
source_config, source_excludes, schema
)
generate.write_configuration(
args.destination_config_filename,
generate.render_configuration(destination_config),
mode=source_config_file_mode,
)
display_result(args)
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def main(): # pragma: no cover
args = parse_arguments(*sys.argv[1:])
logging.basicConfig(level=logging.INFO, format='%(message)s')
config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
if len(config_filenames) == 0:
logger.critical('No files to validate found')
sys.exit(1)
found_issues = False
for config_filename in config_filenames:
try:
validate.parse_configuration(config_filename, validate.schema_filename())
except (ValueError, OSError, validate.Validation_error) as error:
logging.critical('{}: Error parsing configuration file'.format(config_filename))
logging.critical(error)
found_issues = True
if found_issues:
sys.exit(1)
else:
logger.info(
'All given configuration files are valid: {}'.format(', '.join(config_filenames))
)
def run_configuration(config_filename, args): # pragma: no cover
'''
Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
checks.
'''
logger.info('{}: Parsing configuration file'.format(config_filename))
config = validate.parse_configuration(config_filename,
validate.schema_filename())
(location, storage, retention, consistency,
hooks) = (config.get(section_name, {})
for section_name in ('location', 'storage', 'retention',
'consistency', 'hooks'))
try:
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
borg_create.initialize_environment(storage)
if args.create:
hook.execute_hook(hooks.get('before_backup'), config_filename,
'pre-backup')
_run_commands(args, consistency, local_path, location, remote_path,
retention, storage)
if args.create:
hook.execute_hook(hooks.get('after_backup'), config_filename,
'post-backup')
except (OSError, CalledProcessError):
hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
raise
def load_configurations(config_filenames):
'''
Given a sequence of configuration filenames, load and validate each configuration file. Return
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
and sequence of logging.LogRecord instances containing any parse errors.
'''
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
logs = []
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename())
except (ValueError, OSError, validate.Validation_error) as error:
logs.extend([
logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='{}: Error parsing configuration file'.format(
config_filename),
)),
logging.makeLogRecord(
dict(levelno=logging.CRITICAL,
levelname='CRITICAL',
msg=error)),
])
return (configs, logs)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
generate.generate_sample_configuration(args.source_filename,
args.destination_filename,
validate.schema_filename())
print('Generated a sample configuration file at {}.'.format(
args.destination_filename))
print()
if args.source_filename:
print('Merged in the contents of configuration file at {}.'.format(
args.source_filename))
print('To review the changes made, run:')
print()
print(' diff --unified {} {}'.format(args.source_filename,
args.destination_filename))
print()
print(
'Please edit the file to suit your needs. The values are representative.'
)
print('All fields are optional except where indicated.')
print()
print('If you ever need help: https://torsion.org/borgmatic/#issues')
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def collect_configuration_run_summary_logs(config_filenames, args):
'''
Given a sequence of configuration filenames and parsed command-line arguments as an
argparse.ArgumentParser instance, run each configuration file and yield a series of
logging.LogRecord instances containing summary information about each run.
'''
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename()
)
except (ValueError, OSError, validate.Validation_error) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error parsing configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
for config_filename, config in configs.items():
try:
run_configuration(config_filename, config, args)
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
msg='{}: Successfully ran configuration file'.format(config_filename),
)
)
except (ValueError, OSError, CalledProcessError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error running configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if not config_filenames:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
)
)
def mock_config_and_schema(config_yaml):
'''
Set up mocks for the config config YAML string and the default schema so that the code under
test consumes them when parsing the configuration.
'''
config_stream = io.StringIO(config_yaml)
schema_stream = open(module.schema_filename())
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('config.yaml').and_return(
config_stream)
builtins.should_receive('open').with_args('schema.yaml').and_return(
schema_stream)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
print('Generated a sample configuration file at {}.'.format(args.destination_filename))
print()
print('Please edit the file to suit your needs. The values are just representative.')
print('All fields are optional except where indicated.')
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def mock_config_and_schema(config_yaml, schema_yaml=None):
'''
Set up mocks for the given config config YAML string and the schema YAML string, or the default
schema if no schema is provided. The idea is that that the code under test consumes these mocks
when parsing the configuration.
'''
config_stream = io.StringIO(config_yaml)
if schema_yaml is None:
schema_stream = open(module.schema_filename())
else:
schema_stream = io.StringIO(schema_yaml)
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
def mock_config_and_schema(config_yaml, schema_yaml=None):
'''
Set up mocks for the given config config YAML string and the schema YAML string, or the default
schema if no schema is provided. The idea is that that the code under test consumes these mocks
when parsing the configuration.
'''
config_stream = io.StringIO(config_yaml)
if schema_yaml is None:
schema_stream = open(module.schema_filename())
else:
schema_stream = io.StringIO(schema_yaml)
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
generate.generate_sample_configuration(args.destination_filename,
validate.schema_filename())
print('Generated a sample configuration file at {}.'.format(
args.destination_filename))
print()
print(
'Please edit the file to suit your needs. The values are just representative.'
)
print('All fields are optional except where indicated.')
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
config_filenames = tuple(
collect.collect_config_filenames(args.config_paths))
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH,
config_filenames)
if len(config_filenames) == 0:
raise ValueError(
'Error: No configuration files found in: {}'.format(' '.join(
args.config_paths)))
for config_filename in config_filenames:
config = validate.parse_configuration(config_filename,
validate.schema_filename())
(location, storage, retention,
consistency) = (config.get(section_name, {})
for section_name in ('location', 'storage',
'retention', 'consistency'))
remote_path = location.get('remote_path')
create.initialize(storage)
for repository in location['repositories']:
if args.prune:
prune.prune_archives(args.verbosity,
repository,
retention,
remote_path=remote_path)
if args.create:
create.create_archive(
args.verbosity,
repository,
location,
storage,
)
if args.check:
check.check_archives(args.verbosity,
repository,
consistency,
remote_path=remote_path)
except (ValueError, OSError, CalledProcessError) as error:
print(error, file=sys.stderr)
sys.exit(1)
def run_configuration(config_filename, args): # pragma: no cover
'''
Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
checks.
'''
logger.info('{}: Parsing configuration file'.format(config_filename))
config = validate.parse_configuration(config_filename, validate.schema_filename())
(location, storage, retention, consistency, hooks) = (
config.get(section_name, {})
for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
)
try:
remote_path = location.get('remote_path')
create.initialize_environment(storage)
hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
for unexpanded_repository in location['repositories']:
repository = os.path.expanduser(unexpanded_repository)
if args.prune:
logger.info('{}: Pruning archives'.format(repository))
prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
if args.create:
logger.info('{}: Creating archive'.format(repository))
create.create_archive(
args.verbosity,
repository,
location,
storage,
)
if args.check:
logger.info('{}: Running consistency checks'.format(repository))
check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
except (OSError, CalledProcessError):
hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
raise
def run_configuration(config_filename, args): # pragma: no cover
'''
Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
checks.
'''
logger.info('{}: Parsing configuration file'.format(config_filename))
config = validate.parse_configuration(config_filename,
validate.schema_filename())
(location, storage, retention, consistency,
hooks) = (config.get(section_name, {})
for section_name in ('location', 'storage', 'retention',
'consistency', 'hooks'))
try:
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
borg_create.initialize_environment(storage)
hook.execute_hook(hooks.get('before_backup'), config_filename,
'pre-backup')
for unexpanded_repository in location['repositories']:
repository = os.path.expanduser(unexpanded_repository)
dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
if args.prune:
logger.info('{}: Pruning archives{}'.format(
repository, dry_run_label))
borg_prune.prune_archives(
args.verbosity,
args.dry_run,
repository,
storage,
retention,
local_path=local_path,
remote_path=remote_path,
)
if args.create:
logger.info('{}: Creating archive{}'.format(
repository, dry_run_label))
borg_create.create_archive(
args.verbosity,
args.dry_run,
repository,
location,
storage,
local_path=local_path,
remote_path=remote_path,
)
if args.check:
logger.info(
'{}: Running consistency checks'.format(repository))
borg_check.check_archives(
args.verbosity,
repository,
storage,
consistency,
local_path=local_path,
remote_path=remote_path,
)
if args.list:
logger.info('{}: Listing archives'.format(repository))
borg_list.list_archives(
args.verbosity,
repository,
storage,
local_path=local_path,
remote_path=remote_path,
)
if args.info:
logger.info('{}: Displaying summary info for archives'.format(
repository))
borg_info.display_archives_info(
args.verbosity,
repository,
storage,
local_path=local_path,
remote_path=remote_path,
)
hook.execute_hook(hooks.get('after_backup'), config_filename,
'post-backup')
except (OSError, CalledProcessError):
hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
raise
def test_schema_filename_returns_plausable_path():
schema_path = module.schema_filename()
assert schema_path.endswith('/schema.yaml')
def collect_configuration_run_summary_logs(config_filenames, args):
'''
Given a sequence of configuration filenames and parsed command-line arguments as an
argparse.ArgumentParser instance, run each configuration file and yield a series of
logging.LogRecord instances containing summary information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename()
)
except (ValueError, OSError, validate.Validation_error) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error parsing configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
json_results = []
for config_filename, config in configs.items():
try:
json_results.extend(list(run_configuration(config_filename, config, args)))
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
msg='{}: Successfully ran configuration file'.format(config_filename),
)
)
except (ValueError, OSError, CalledProcessError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error running configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if json_results:
sys.stdout.write(json.dumps(json_results))
if not config_filenames:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
)
)
def test_schema_filename_returns_plausable_path():
schema_path = module.schema_filename()
assert schema_path.endswith('/schema.yaml')
