def clean_backups(self): rotation_fn = os.path.dirname(os.path.realpath(__file__)) + '/rotation.ini' res = rotate_backups.load_config_file(rotation_fn) for location, rotation_scheme, options in res: options['prefer_recent'] = True rotate_backups.RotateBackups(rotation_scheme=rotation_scheme, **options).rotate_backups(location) fix_latest_link(self._backup_dir, self._latest_dir, container=DockerContainer.BACKUP)
def test_filename_patterns(self): """Test support for filename patterns in configuration files.""" with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: for subdirectory in 'laptop', 'vps': os.makedirs(os.path.join(root, subdirectory)) config_file = os.path.join(root, 'rotate-backups.ini') parser = configparser.RawConfigParser() pattern = os.path.join(root, '*') parser.add_section(pattern) parser.set(pattern, 'daily', '7') parser.set(pattern, 'weekly', '4') parser.set(pattern, 'monthly', 'always') with open(config_file, 'w') as handle: parser.write(handle) # Check that the configured rotation scheme is applied. default_scheme = dict(monthly='always') program = RotateBackups(config_file=config_file, rotation_scheme=default_scheme) program.load_config_file(os.path.join(root, 'laptop')) assert program.rotation_scheme != default_scheme # Check that the available locations are matched. available_locations = [ location for location, rotation_scheme, options in load_config_file(config_file) ] assert len(available_locations) == 2 assert any(location.directory == os.path.join(root, 'laptop') for location in available_locations) assert any(location.directory == os.path.join(root, 'vps') for location in available_locations)
def test_argument_validation(self): """Test argument validation.""" # Test that an invalid ionice scheduling class causes an error to be reported. returncode, output = run_cli(main, '--ionice=unsupported-class') assert returncode != 0 # Test that an invalid rotation scheme causes an error to be reported. returncode, output = run_cli(main, '--hourly=not-a-number') assert returncode != 0 # Argument validation tests that require an empty directory. with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: # Test that non-existing directories cause an error to be reported. returncode, output = run_cli(main, os.path.join(root, 'does-not-exist')) assert returncode != 0 # Test that loading of a custom configuration file raises an # exception when the configuration file cannot be loaded. self.assertRaises( ValueError, lambda: list( load_config_file(os.path.join(root, 'rotate-backups.ini'))) ) # Test that an empty rotation scheme raises an exception. self.create_sample_backup_set(root) self.assertRaises( ValueError, lambda: RotateBackups(rotation_scheme={}).rotate_backups(root)) # Argument validation tests that assume the current user isn't root. if os.getuid() != 0: # I'm being lazy and will assume that this test suite will only be # run on systems where users other than root do not have access to # /root. returncode, output = run_cli(main, '-n', '/root') assert returncode != 0
def test_filename_patterns(self): """Test support for filename patterns in configuration files.""" with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: for subdirectory in 'laptop', 'vps': os.makedirs(os.path.join(root, subdirectory)) config_file = os.path.join(root, 'rotate-backups.ini') parser = configparser.RawConfigParser() pattern = os.path.join(root, '*') parser.add_section(pattern) parser.set(pattern, 'daily', '7') parser.set(pattern, 'weekly', '4') parser.set(pattern, 'monthly', 'always') with open(config_file, 'w') as handle: parser.write(handle) # Check that the configured rotation scheme is applied. default_scheme = dict(monthly='always') program = RotateBackups(config_file=config_file, rotation_scheme=default_scheme) program.load_config_file(os.path.join(root, 'laptop')) assert program.rotation_scheme != default_scheme # Check that the available locations are matched. available_locations = [ location for location, rotation_scheme, options in load_config_file(config_file) ] assert len(available_locations) == 2 assert any(location.directory == os.path.join(root, 'laptop') for location in available_locations) assert any(location.directory == os.path.join(root, 'vps') for location in available_locations)
def test_argument_validation(self): """Test argument validation.""" # Test that an invalid ionice scheduling class causes an error to be reported. returncode, output = run_cli(main, '--ionice=unsupported-class') assert returncode != 0 # Test that an invalid rotation scheme causes an error to be reported. returncode, output = run_cli(main, '--hourly=not-a-number') assert returncode != 0 # Argument validation tests that require an empty directory. with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: # Test that non-existing directories cause an error to be reported. returncode, output = run_cli(main, os.path.join(root, 'does-not-exist')) assert returncode != 0 # Test that loading of a custom configuration file raises an # exception when the configuration file cannot be loaded. self.assertRaises(ValueError, lambda: list(load_config_file(os.path.join(root, 'rotate-backups.ini')))) # Test that an empty rotation scheme raises an exception. self.create_sample_backup_set(root) self.assertRaises(ValueError, lambda: RotateBackups(rotation_scheme={}).rotate_backups(root)) # Argument validation tests that assume the current user isn't root. if os.getuid() != 0: # I'm being lazy and will assume that this test suite will only be # run on systems where users other than root do not have access to # /root. returncode, output = run_cli(main, '-n', '/root') assert returncode != 0
def clean_backups(self): rotation_fn = os.path.dirname( os.path.realpath(__file__)) + '/rotation.ini' res = rotate_backups.load_config_file(rotation_fn) for location, rotation_scheme, options in res: options['prefer_recent'] = True rotate_backups.RotateBackups(rotation_scheme=rotation_scheme, **options).rotate_backups(location)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. rotation_scheme = {} kw = dict(include_list=[], exclude_list=[]) parallel = False use_sudo = False # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [ 'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-M', '--minutely'): rotation_scheme['minutely'] = coerce_retention_period(value) elif option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): kw['include_list'].append(value) elif option in ('-x', '--exclude'): kw['exclude_list'].append(value) elif option in ('-j', '--parallel'): parallel = True elif option in ('-p', '--prefer-recent'): kw['prefer_recent'] = True elif option in ('-r', '--relaxed'): kw['strict'] = False elif option in ('-i', '--ionice'): value = validate_ionice_class(value.lower().strip()) kw['io_scheduling_class'] = value elif option in ('-c', '--config'): kw['config_file'] = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) kw['dry_run'] = True elif option in ('-C', '--removal-command'): removal_command = shlex.split(value) logger.info("Using custom removal command: %s", removal_command) kw['removal_command'] = removal_command elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. location_source = 'command line arguments' selected_locations.extend( coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. location_source = 'configuration file' selected_locations.extend( location for location, rotation_scheme, options in load_config_file( configuration_file=kw.get('config_file'), expand=True)) # Inform the user which location(s) will be rotated. if selected_locations: logger.verbose("Selected %s based on %s:", pluralize(len(selected_locations), "location"), location_source) for number, location in enumerate(selected_locations, start=1): logger.verbose(" %i. %s", number, location) else: # Show the usage message when no directories are given nor configured. logger.verbose("No location(s) to rotate selected.") usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. program = RotateBackups(rotation_scheme, **kw) if parallel: program.rotate_concurrent(*selected_locations) else: for location in selected_locations: program.rotate_backups(location)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. config_file = None dry_run = False exclude_list = [] include_list = [] io_scheduling_class = None rotation_scheme = {} use_sudo = False strict = True # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:ri:c:r:unvqh', [ 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): include_list.append(value) elif option in ('-x', '--exclude'): exclude_list.append(value) elif option in ('-r', '--relaxed'): strict = False elif option in ('-i', '--ionice'): value = value.lower().strip() expected = ('idle', 'best-effort', 'realtime') if value not in expected: msg = "Invalid I/O scheduling class! (got %r while valid options are %s)" raise Exception(msg % (value, concatenate(expected))) io_scheduling_class = value elif option in ('-c', '--config'): config_file = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) dry_run = True elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.debug("Parsed rotation scheme: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. selected_locations.extend(location for location, rotation_scheme, options in load_config_file(config_file)) # Show the usage message when no directories are given nor configured. if not selected_locations: usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. for location in selected_locations: RotateBackups( rotation_scheme=rotation_scheme, include_list=include_list, exclude_list=exclude_list, io_scheduling_class=io_scheduling_class, dry_run=dry_run, config_file=config_file, strict=strict, ).rotate_backups(location)
def main(): """Command line interface for the ``rotate-backups-s3`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. aws_access_key_id = None aws_secret_access_key = None aws_host = 's3.amazonaws.com' config_file = None dry_run = False exclude_list = [] include_list = [] rotation_scheme = {} prefer_recent = False # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'U:P:H:d:w:m:y:I:x:c:nvhp', [ 'aws-access-key-id=', 'aws-secret-access-key=', 'aws-host=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'config=', 'dry-run', 'verbose', 'help', 'prefer-recent', ]) for option, value in options: if option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): include_list.append(value) elif option in ('-x', '--exclude'): exclude_list.append(value) elif option in ('-c', '--config'): config_file = parse_path(value) elif option in ('-U', '--aws-access-key-id'): aws_access_key_id = value elif option in ('-P', '--aws-secret-access-key'): aws_secret_access_key = value elif option in ('--aws-host'): aws_host = value elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) dry_run = True elif option in ('-p', '--prefer-recent'): prefer_recent = True elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.debug("Parsed rotation scheme: %s", rotation_scheme) # If no arguments are given but the system has a configuration file # then the backups in the configured directories are rotated. if not arguments: arguments.extend(s3path for s3path, _, _ in load_config_file(config_file)) # Show the usage message when no directories are given nor configured. if not arguments: usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the given or configured directories. for s3path in arguments: prefix = '' bucket = s3path[5:] if s3path.startswith('s3://') else s3path pos = bucket.find('/') if pos != -1: prefix = bucket[pos:].strip('/') bucket = bucket[:pos] S3RotateBackups( rotation_scheme=rotation_scheme, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_host=aws_host, include_list=include_list, exclude_list=exclude_list, dry_run=dry_run, prefer_recent=prefer_recent, ).rotate_backups(bucket, prefix)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. rotation_scheme = {} kw = dict(include_list=[], exclude_list=[]) parallel = False use_sudo = False # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [ 'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-M', '--minutely'): rotation_scheme['minutely'] = coerce_retention_period(value) elif option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): kw['include_list'].append(value) elif option in ('-x', '--exclude'): kw['exclude_list'].append(value) elif option in ('-j', '--parallel'): parallel = True elif option in ('-p', '--prefer-recent'): kw['prefer_recent'] = True elif option in ('-r', '--relaxed'): kw['strict'] = False elif option in ('-i', '--ionice'): value = validate_ionice_class(value.lower().strip()) kw['io_scheduling_class'] = value elif option in ('-c', '--config'): kw['config_file'] = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) kw['dry_run'] = True elif option in ('-C', '--removal-command'): removal_command = shlex.split(value) logger.info("Using custom removal command: %s", removal_command) kw['removal_command'] = removal_command elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. location_source = 'command line arguments' selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. location_source = 'configuration file' selected_locations.extend( location for location, rotation_scheme, options in load_config_file(configuration_file=kw.get('config_file'), expand=True) ) # Inform the user which location(s) will be rotated. if selected_locations: logger.verbose("Selected %s based on %s:", pluralize(len(selected_locations), "location"), location_source) for number, location in enumerate(selected_locations, start=1): logger.verbose(" %i. %s", number, location) else: # Show the usage message when no directories are given nor configured. logger.verbose("No location(s) to rotate selected.") usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. program = RotateBackups(rotation_scheme, **kw) if parallel: program.rotate_concurrent(*selected_locations) else: for location in selected_locations: program.rotate_backups(location)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. config_file = None dry_run = False exclude_list = [] include_list = [] io_scheduling_class = None rotation_scheme = {} use_sudo = False strict = True # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:ri:c:r:unvqh', [ 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): include_list.append(value) elif option in ('-x', '--exclude'): exclude_list.append(value) elif option in ('-r', '--relaxed'): strict = False elif option in ('-i', '--ionice'): value = value.lower().strip() expected = ('idle', 'best-effort', 'realtime') if value not in expected: msg = "Invalid I/O scheduling class! (got %r while valid options are %s)" raise Exception(msg % (value, concatenate(expected))) io_scheduling_class = value elif option in ('-c', '--config'): config_file = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) dry_run = True elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.debug("Parsed rotation scheme: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. selected_locations.extend( coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. selected_locations.extend(location for location, rotation_scheme, options in load_config_file(config_file)) # Show the usage message when no directories are given nor configured. if not selected_locations: usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. for location in selected_locations: RotateBackups( rotation_scheme=rotation_scheme, include_list=include_list, exclude_list=exclude_list, io_scheduling_class=io_scheduling_class, dry_run=dry_run, config_file=config_file, strict=strict, ).rotate_backups(location)