def clean_backups(self):
rotation_fn = os.path.dirname(os.path.realpath(__file__)) + '/rotation.ini'
res = rotate_backups.load_config_file(rotation_fn)
for location, rotation_scheme, options in res:
options['prefer_recent'] = True
rotate_backups.RotateBackups(rotation_scheme=rotation_scheme, **options).rotate_backups(location)
fix_latest_link(self._backup_dir, self._latest_dir, container=DockerContainer.BACKUP)
def test_filename_patterns(self):
"""Test support for filename patterns in configuration files."""
with TemporaryDirectory(prefix='rotate-backups-',
suffix='-test-suite') as root:
for subdirectory in 'laptop', 'vps':
os.makedirs(os.path.join(root, subdirectory))
config_file = os.path.join(root, 'rotate-backups.ini')
parser = configparser.RawConfigParser()
pattern = os.path.join(root, '*')
parser.add_section(pattern)
parser.set(pattern, 'daily', '7')
parser.set(pattern, 'weekly', '4')
parser.set(pattern, 'monthly', 'always')
with open(config_file, 'w') as handle:
parser.write(handle)
# Check that the configured rotation scheme is applied.
default_scheme = dict(monthly='always')
program = RotateBackups(config_file=config_file,
rotation_scheme=default_scheme)
program.load_config_file(os.path.join(root, 'laptop'))
assert program.rotation_scheme != default_scheme
# Check that the available locations are matched.
available_locations = [
location for location, rotation_scheme, options in
load_config_file(config_file)
]
assert len(available_locations) == 2
assert any(location.directory == os.path.join(root, 'laptop')
for location in available_locations)
assert any(location.directory == os.path.join(root, 'vps')
for location in available_locations)
def test_argument_validation(self):
"""Test argument validation."""
# Test that an invalid ionice scheduling class causes an error to be reported.
returncode, output = run_cli(main, '--ionice=unsupported-class')
assert returncode != 0
# Test that an invalid rotation scheme causes an error to be reported.
returncode, output = run_cli(main, '--hourly=not-a-number')
assert returncode != 0
# Argument validation tests that require an empty directory.
with TemporaryDirectory(prefix='rotate-backups-',
suffix='-test-suite') as root:
# Test that non-existing directories cause an error to be reported.
returncode, output = run_cli(main,
os.path.join(root, 'does-not-exist'))
assert returncode != 0
# Test that loading of a custom configuration file raises an
# exception when the configuration file cannot be loaded.
self.assertRaises(
ValueError, lambda: list(
load_config_file(os.path.join(root, 'rotate-backups.ini')))
)
# Test that an empty rotation scheme raises an exception.
self.create_sample_backup_set(root)
self.assertRaises(
ValueError,
lambda: RotateBackups(rotation_scheme={}).rotate_backups(root))
# Argument validation tests that assume the current user isn't root.
if os.getuid() != 0:
# I'm being lazy and will assume that this test suite will only be
# run on systems where users other than root do not have access to
# /root.
returncode, output = run_cli(main, '-n', '/root')
assert returncode != 0
def test_filename_patterns(self):
"""Test support for filename patterns in configuration files."""
with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
for subdirectory in 'laptop', 'vps':
os.makedirs(os.path.join(root, subdirectory))
config_file = os.path.join(root, 'rotate-backups.ini')
parser = configparser.RawConfigParser()
pattern = os.path.join(root, '*')
parser.add_section(pattern)
parser.set(pattern, 'daily', '7')
parser.set(pattern, 'weekly', '4')
parser.set(pattern, 'monthly', 'always')
with open(config_file, 'w') as handle:
parser.write(handle)
# Check that the configured rotation scheme is applied.
default_scheme = dict(monthly='always')
program = RotateBackups(config_file=config_file, rotation_scheme=default_scheme)
program.load_config_file(os.path.join(root, 'laptop'))
assert program.rotation_scheme != default_scheme
# Check that the available locations are matched.
available_locations = [
location for location, rotation_scheme, options
in load_config_file(config_file)
]
assert len(available_locations) == 2
assert any(location.directory == os.path.join(root, 'laptop') for location in available_locations)
assert any(location.directory == os.path.join(root, 'vps') for location in available_locations)
def test_argument_validation(self):
"""Test argument validation."""
# Test that an invalid ionice scheduling class causes an error to be reported.
returncode, output = run_cli(main, '--ionice=unsupported-class')
assert returncode != 0
# Test that an invalid rotation scheme causes an error to be reported.
returncode, output = run_cli(main, '--hourly=not-a-number')
assert returncode != 0
# Argument validation tests that require an empty directory.
with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
# Test that non-existing directories cause an error to be reported.
returncode, output = run_cli(main, os.path.join(root, 'does-not-exist'))
assert returncode != 0
# Test that loading of a custom configuration file raises an
# exception when the configuration file cannot be loaded.
self.assertRaises(ValueError, lambda: list(load_config_file(os.path.join(root, 'rotate-backups.ini'))))
# Test that an empty rotation scheme raises an exception.
self.create_sample_backup_set(root)
self.assertRaises(ValueError, lambda: RotateBackups(rotation_scheme={}).rotate_backups(root))
# Argument validation tests that assume the current user isn't root.
if os.getuid() != 0:
# I'm being lazy and will assume that this test suite will only be
# run on systems where users other than root do not have access to
# /root.
returncode, output = run_cli(main, '-n', '/root')
assert returncode != 0
def clean_backups(self):
rotation_fn = os.path.dirname(
os.path.realpath(__file__)) + '/rotation.ini'
res = rotate_backups.load_config_file(rotation_fn)
for location, rotation_scheme, options in res:
options['prefer_recent'] = True
rotate_backups.RotateBackups(rotation_scheme=rotation_scheme,
**options).rotate_backups(location)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh',
[
'minutely=',
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'parallel',
'prefer-recent',
'relaxed',
'ionice=',
'config=',
'use-sudo',
'dry-run',
'removal-command=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s",
removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s",
rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(
coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location
for location, rotation_scheme, options in load_config_file(
configuration_file=kw.get('config_file'), expand=True))
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
config_file = None
dry_run = False
exclude_list = []
include_list = []
io_scheduling_class = None
rotation_scheme = {}
use_sudo = False
strict = True
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:ri:c:r:unvqh', [
'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=',
'exclude=', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run',
'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-r', '--relaxed'):
strict = False
elif option in ('-i', '--ionice'):
value = value.lower().strip()
expected = ('idle', 'best-effort', 'realtime')
if value not in expected:
msg = "Invalid I/O scheduling class! (got %r while valid options are %s)"
raise Exception(msg % (value, concatenate(expected)))
io_scheduling_class = value
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
selected_locations.extend(location for location, rotation_scheme, options in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not selected_locations:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
for location in selected_locations:
RotateBackups(
rotation_scheme=rotation_scheme,
include_list=include_list,
exclude_list=exclude_list,
io_scheduling_class=io_scheduling_class,
dry_run=dry_run,
config_file=config_file,
strict=strict,
).rotate_backups(location)
def main():
"""Command line interface for the ``rotate-backups-s3`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
aws_access_key_id = None
aws_secret_access_key = None
aws_host = 's3.amazonaws.com'
config_file = None
dry_run = False
exclude_list = []
include_list = []
rotation_scheme = {}
prefer_recent = False
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'U:P:H:d:w:m:y:I:x:c:nvhp', [
'aws-access-key-id=',
'aws-secret-access-key=',
'aws-host=',
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'config=',
'dry-run',
'verbose',
'help',
'prefer-recent',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-U', '--aws-access-key-id'):
aws_access_key_id = value
elif option in ('-P', '--aws-secret-access-key'):
aws_secret_access_key = value
elif option in ('--aws-host'):
aws_host = value
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
dry_run = True
elif option in ('-p', '--prefer-recent'):
prefer_recent = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
# If no arguments are given but the system has a configuration file
# then the backups in the configured directories are rotated.
if not arguments:
arguments.extend(s3path
for s3path, _, _ in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not arguments:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the given or configured directories.
for s3path in arguments:
prefix = ''
bucket = s3path[5:] if s3path.startswith('s3://') else s3path
pos = bucket.find('/')
if pos != -1:
prefix = bucket[pos:].strip('/')
bucket = bucket[:pos]
S3RotateBackups(
rotation_scheme=rotation_scheme,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_host=aws_host,
include_list=include_list,
exclude_list=exclude_list,
dry_run=dry_run,
prefer_recent=prefer_recent,
).rotate_backups(bucket, prefix)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [
'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',
'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s", removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location for location, rotation_scheme, options
in load_config_file(configuration_file=kw.get('config_file'), expand=True)
)
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
config_file = None
dry_run = False
exclude_list = []
include_list = []
io_scheduling_class = None
rotation_scheme = {}
use_sudo = False
strict = True
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'H:d:w:m:y:I:x:ri:c:r:unvqh', [
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'relaxed',
'ionice=',
'config=',
'use-sudo',
'dry-run',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-r', '--relaxed'):
strict = False
elif option in ('-i', '--ionice'):
value = value.lower().strip()
expected = ('idle', 'best-effort', 'realtime')
if value not in expected:
msg = "Invalid I/O scheduling class! (got %r while valid options are %s)"
raise Exception(msg % (value, concatenate(expected)))
io_scheduling_class = value
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
selected_locations.extend(
coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
selected_locations.extend(location for location, rotation_scheme,
options in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not selected_locations:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
for location in selected_locations:
RotateBackups(
rotation_scheme=rotation_scheme,
include_list=include_list,
exclude_list=exclude_list,
io_scheduling_class=io_scheduling_class,
dry_run=dry_run,
config_file=config_file,
strict=strict,
).rotate_backups(location)