def rotate_snapshots(self):
"""
Rotate system backup snapshots using :mod:`.rotate_backups`.
:raises: The following exceptions can be raised:
- :exc:`.DestinationContextUnavailable`, refer
to :attr:`destination_context` for details.
- :exc:`.ParentDirectoryUnavailable`, refer
to :attr:`.parent_directory` for details.
- Any exceptions raised by :mod:`.rotate_backups`.
The values of the :attr:`dry_run`, :attr:`ionice` and
:attr:`rotation_scheme` properties are passed on to the
:class:`~rotate_backups.RotateBackups` class.
"""
helper = RotateBackups(
dry_run=self.dry_run,
io_scheduling_class=self.ionice,
rotation_scheme=self.rotation_scheme,
)
helper.rotate_backups(
Location(
context=self.destination_context,
directory=self.destination.parent_directory,
))
def test_removal_command(self):
"""Test that the removal command can be customized."""
with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
today = datetime.datetime.now()
for date in today, (today - datetime.timedelta(hours=24)):
os.mkdir(os.path.join(root, date.strftime('%Y-%m-%d')))
program = RotateBackups(removal_command=['rmdir'], rotation_scheme=dict(monthly='always'))
commands = program.rotate_backups(root, prepare=True)
assert any(cmd.command_line[0] == 'rmdir' for cmd in commands)
def test_removal_command(self):
"""Test that the removal command can be customized."""
with TemporaryDirectory(prefix='rotate-backups-',
suffix='-test-suite') as root:
today = datetime.datetime.now()
for date in today, (today - datetime.timedelta(hours=24)):
os.mkdir(os.path.join(root, date.strftime('%Y-%m-%d')))
program = RotateBackups(removal_command=['rmdir'],
rotation_scheme=dict(monthly='always'))
commands = program.rotate_backups(root, prepare=True)
assert any(cmd.command_line[0] == 'rmdir' for cmd in commands)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh',
[
'minutely=',
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'parallel',
'prefer-recent',
'relaxed',
'ionice=',
'config=',
'use-sudo',
'dry-run',
'removal-command=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s",
removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s",
rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(
coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location
for location, rotation_scheme, options in load_config_file(
configuration_file=kw.get('config_file'), expand=True))
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [
'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',
'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s", removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location for location, rotation_scheme, options
in load_config_file(configuration_file=kw.get('config_file'), expand=True)
)
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def handle_task(task, args, config):
# l = make_logger(args)
# l = logging.getLogger(task["name"])
ll = make_logger(args, "shbackup - " + task["name"].upper())
start = datetime.now()
conn = check_requirements(ll)
# ll = LoggerAdapter(l, task["name"])
try:
auth = task["auth"]
ll.info("{} start".format(task["name"]))
init_cmds = task["ftp_init_cmds"] if "ftp_init_cmds" in task and type(
task["ftp_init_cmds"]) == list else []
conn.connect(ll,
host=auth["host"],
user=auth["user"],
pwd=auth["pass"],
port=auth["port"] if "port" in auth else None,
init_cmds=init_cmds,
verbose=args.verbose)
local_dir = task["local_dir"]
remote_dir = task["remote_dir"]
sql_dir = os.path.join(local_dir, "db")
files_dir = os.path.join(local_dir, "files")
current_dir = os.path.join(files_dir, "current")
versions_dir = os.path.join(files_dir, "versions")
do_database = not args.no_db and ("mysql_config" in task
and task["mysql_config"] != False)
do_files = not args.no_files and ("sync_files" in task
and task["sync_files"] != False)
# print(task["name"], do_database, do_files)
# return True
for d in [sql_dir, current_dir, versions_dir]:
os.system("mkdir -p {}".format(d))
if do_database:
dump = get_mysql_dump(ll, args, task, conn)
mysql_dump_filename = os.path.join(
sql_dir, "{}_mysql_{}.sql.gz".format(
task["name"],
datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))
with gzip.open(
mysql_dump_filename,
"wb",
) as f:
ll.debug("writing to {}".format(mysql_dump_filename))
f.write(dump)
if do_files:
ll.info("Syncing remote directory to current cache")
exs = task["excludes"] if type(task["excludes"]) == list else []
exs.append("mysqldump/")
with conn.cwd(remote_dir):
conn.mirror("./",
current_dir,
parallel=int(task["max_conn"]),
exclude=exs)
files_version_filename = os.path.join(
versions_dir, "{}_files_{}.tar.gz".format(
task["name"],
datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))
ll.info("Building tarball archive")
with tarfile.open(files_version_filename, "w:gz") as tar:
ll.debug("writing to {}".format(files_version_filename))
tar.add(current_dir, arcname=os.path.basename(current_dir))
# if do_database:
ll.info("Rotating database versions")
# db_retention = config["default_db_retention"] if "default_db_retention" in config else {}
# db_retention.update(task["db_retention"] if "db_retention" in task else {})
if "db_retention" in task:
db_retention = task["db_retention"]
else:
db_retention = config["default_db_retention"]
ll.debug("db retention: " + repr(db_retention))
db_rotator = RotateBackups(db_retention,
dry_run=False,
prefer_recent=True,
strict=False)
dbloc = coerce_location(sql_dir)
db_rotator.rotate_backups(dbloc)
# if do_files:
ll.info("Rotating files versions")
# files_retention = config["default_files_retention"] if "default_files_retention" in config else {}
# files_retention.update(task["files_retention"] if "files_retention" in task else {})
if "files_retention" in task:
files_retention = task["files_retention"]
else:
files_retention = config["default_files_retention"]
ll.debug("files retention: " + repr(files_retention))
files_rotator = RotateBackups(files_retention,
dry_run=False,
prefer_recent=True,
strict=False)
filesloc = coerce_location(versions_dir)
files_rotator.rotate_backups(filesloc)
conn.close()
if "post_cmd" in task and task["post_cmd"] and not args.skip_post_cmds:
try:
cmd = task["post_cmd"]
ll.info("Executing post command")
ll.debug(cmd)
with open(os.devnull, 'w') as FNULL:
subprocess.check_call(cmd,
shell=True,
stdout=FNULL,
stderr=subprocess.STDOUT)
except:
ll.error("Error when executing post command")
delta = datetime.now() - start
ll.info("{} completed in {}s".format(task["name"], delta.seconds))
return True
except KeyboardInterrupt:
print("KeyboardInterrupt caught while processing task {}".format(
task["name"]))
sys.exit(0)
return False
except pexpect.exceptions.TIMEOUT:
ll.error("Timeout caught")
return False
except Exception as e:
ll.critical("Error while processing {}. continuing".format(
task["name"]))
ll.critical(str(e))
if ll.getEffectiveLevel() <= logging.DEBUG:
traceback.print_exc()
return False