def rotate_snapshots(self): """ Rotate system backup snapshots using :mod:`.rotate_backups`. :raises: The following exceptions can be raised: - :exc:`.DestinationContextUnavailable`, refer to :attr:`destination_context` for details. - :exc:`.ParentDirectoryUnavailable`, refer to :attr:`.parent_directory` for details. - Any exceptions raised by :mod:`.rotate_backups`. The values of the :attr:`dry_run`, :attr:`ionice` and :attr:`rotation_scheme` properties are passed on to the :class:`~rotate_backups.RotateBackups` class. """ helper = RotateBackups( dry_run=self.dry_run, io_scheduling_class=self.ionice, rotation_scheme=self.rotation_scheme, ) helper.rotate_backups( Location( context=self.destination_context, directory=self.destination.parent_directory, ))
def test_removal_command(self): """Test that the removal command can be customized.""" with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: today = datetime.datetime.now() for date in today, (today - datetime.timedelta(hours=24)): os.mkdir(os.path.join(root, date.strftime('%Y-%m-%d'))) program = RotateBackups(removal_command=['rmdir'], rotation_scheme=dict(monthly='always')) commands = program.rotate_backups(root, prepare=True) assert any(cmd.command_line[0] == 'rmdir' for cmd in commands)
def test_removal_command(self): """Test that the removal command can be customized.""" with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: today = datetime.datetime.now() for date in today, (today - datetime.timedelta(hours=24)): os.mkdir(os.path.join(root, date.strftime('%Y-%m-%d'))) program = RotateBackups(removal_command=['rmdir'], rotation_scheme=dict(monthly='always')) commands = program.rotate_backups(root, prepare=True) assert any(cmd.command_line[0] == 'rmdir' for cmd in commands)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. rotation_scheme = {} kw = dict(include_list=[], exclude_list=[]) parallel = False use_sudo = False # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [ 'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-M', '--minutely'): rotation_scheme['minutely'] = coerce_retention_period(value) elif option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): kw['include_list'].append(value) elif option in ('-x', '--exclude'): kw['exclude_list'].append(value) elif option in ('-j', '--parallel'): parallel = True elif option in ('-p', '--prefer-recent'): kw['prefer_recent'] = True elif option in ('-r', '--relaxed'): kw['strict'] = False elif option in ('-i', '--ionice'): value = validate_ionice_class(value.lower().strip()) kw['io_scheduling_class'] = value elif option in ('-c', '--config'): kw['config_file'] = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) kw['dry_run'] = True elif option in ('-C', '--removal-command'): removal_command = shlex.split(value) logger.info("Using custom removal command: %s", removal_command) kw['removal_command'] = removal_command elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. location_source = 'command line arguments' selected_locations.extend( coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. location_source = 'configuration file' selected_locations.extend( location for location, rotation_scheme, options in load_config_file( configuration_file=kw.get('config_file'), expand=True)) # Inform the user which location(s) will be rotated. if selected_locations: logger.verbose("Selected %s based on %s:", pluralize(len(selected_locations), "location"), location_source) for number, location in enumerate(selected_locations, start=1): logger.verbose(" %i. %s", number, location) else: # Show the usage message when no directories are given nor configured. logger.verbose("No location(s) to rotate selected.") usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. program = RotateBackups(rotation_scheme, **kw) if parallel: program.rotate_concurrent(*selected_locations) else: for location in selected_locations: program.rotate_backups(location)
def main(): """Command line interface for the ``rotate-backups`` program.""" coloredlogs.install(syslog=True) # Command line option defaults. rotation_scheme = {} kw = dict(include_list=[], exclude_list=[]) parallel = False use_sudo = False # Internal state. selected_locations = [] # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [ 'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-M', '--minutely'): rotation_scheme['minutely'] = coerce_retention_period(value) elif option in ('-H', '--hourly'): rotation_scheme['hourly'] = coerce_retention_period(value) elif option in ('-d', '--daily'): rotation_scheme['daily'] = coerce_retention_period(value) elif option in ('-w', '--weekly'): rotation_scheme['weekly'] = coerce_retention_period(value) elif option in ('-m', '--monthly'): rotation_scheme['monthly'] = coerce_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = coerce_retention_period(value) elif option in ('-I', '--include'): kw['include_list'].append(value) elif option in ('-x', '--exclude'): kw['exclude_list'].append(value) elif option in ('-j', '--parallel'): parallel = True elif option in ('-p', '--prefer-recent'): kw['prefer_recent'] = True elif option in ('-r', '--relaxed'): kw['strict'] = False elif option in ('-i', '--ionice'): value = validate_ionice_class(value.lower().strip()) kw['io_scheduling_class'] = value elif option in ('-c', '--config'): kw['config_file'] = parse_path(value) elif option in ('-u', '--use-sudo'): use_sudo = True elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) kw['dry_run'] = True elif option in ('-C', '--removal-command'): removal_command = shlex.split(value) logger.info("Using custom removal command: %s", removal_command) kw['removal_command'] = removal_command elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option! (programming error)" if rotation_scheme: logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme) if arguments: # Rotation of the locations given on the command line. location_source = 'command line arguments' selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments) else: # Rotation of all configured locations. location_source = 'configuration file' selected_locations.extend( location for location, rotation_scheme, options in load_config_file(configuration_file=kw.get('config_file'), expand=True) ) # Inform the user which location(s) will be rotated. if selected_locations: logger.verbose("Selected %s based on %s:", pluralize(len(selected_locations), "location"), location_source) for number, location in enumerate(selected_locations, start=1): logger.verbose(" %i. %s", number, location) else: # Show the usage message when no directories are given nor configured. logger.verbose("No location(s) to rotate selected.") usage(__doc__) return except Exception as e: logger.error("%s", e) sys.exit(1) # Rotate the backups in the selected directories. program = RotateBackups(rotation_scheme, **kw) if parallel: program.rotate_concurrent(*selected_locations) else: for location in selected_locations: program.rotate_backups(location)
def handle_task(task, args, config): # l = make_logger(args) # l = logging.getLogger(task["name"]) ll = make_logger(args, "shbackup - " + task["name"].upper()) start = datetime.now() conn = check_requirements(ll) # ll = LoggerAdapter(l, task["name"]) try: auth = task["auth"] ll.info("{} start".format(task["name"])) init_cmds = task["ftp_init_cmds"] if "ftp_init_cmds" in task and type( task["ftp_init_cmds"]) == list else [] conn.connect(ll, host=auth["host"], user=auth["user"], pwd=auth["pass"], port=auth["port"] if "port" in auth else None, init_cmds=init_cmds, verbose=args.verbose) local_dir = task["local_dir"] remote_dir = task["remote_dir"] sql_dir = os.path.join(local_dir, "db") files_dir = os.path.join(local_dir, "files") current_dir = os.path.join(files_dir, "current") versions_dir = os.path.join(files_dir, "versions") do_database = not args.no_db and ("mysql_config" in task and task["mysql_config"] != False) do_files = not args.no_files and ("sync_files" in task and task["sync_files"] != False) # print(task["name"], do_database, do_files) # return True for d in [sql_dir, current_dir, versions_dir]: os.system("mkdir -p {}".format(d)) if do_database: dump = get_mysql_dump(ll, args, task, conn) mysql_dump_filename = os.path.join( sql_dir, "{}_mysql_{}.sql.gz".format( task["name"], datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))) with gzip.open( mysql_dump_filename, "wb", ) as f: ll.debug("writing to {}".format(mysql_dump_filename)) f.write(dump) if do_files: ll.info("Syncing remote directory to current cache") exs = task["excludes"] if type(task["excludes"]) == list else [] exs.append("mysqldump/") with conn.cwd(remote_dir): conn.mirror("./", current_dir, parallel=int(task["max_conn"]), exclude=exs) files_version_filename = os.path.join( versions_dir, "{}_files_{}.tar.gz".format( task["name"], datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))) ll.info("Building tarball archive") with tarfile.open(files_version_filename, "w:gz") as tar: ll.debug("writing to {}".format(files_version_filename)) tar.add(current_dir, arcname=os.path.basename(current_dir)) # if do_database: ll.info("Rotating database versions") # db_retention = config["default_db_retention"] if "default_db_retention" in config else {} # db_retention.update(task["db_retention"] if "db_retention" in task else {}) if "db_retention" in task: db_retention = task["db_retention"] else: db_retention = config["default_db_retention"] ll.debug("db retention: " + repr(db_retention)) db_rotator = RotateBackups(db_retention, dry_run=False, prefer_recent=True, strict=False) dbloc = coerce_location(sql_dir) db_rotator.rotate_backups(dbloc) # if do_files: ll.info("Rotating files versions") # files_retention = config["default_files_retention"] if "default_files_retention" in config else {} # files_retention.update(task["files_retention"] if "files_retention" in task else {}) if "files_retention" in task: files_retention = task["files_retention"] else: files_retention = config["default_files_retention"] ll.debug("files retention: " + repr(files_retention)) files_rotator = RotateBackups(files_retention, dry_run=False, prefer_recent=True, strict=False) filesloc = coerce_location(versions_dir) files_rotator.rotate_backups(filesloc) conn.close() if "post_cmd" in task and task["post_cmd"] and not args.skip_post_cmds: try: cmd = task["post_cmd"] ll.info("Executing post command") ll.debug(cmd) with open(os.devnull, 'w') as FNULL: subprocess.check_call(cmd, shell=True, stdout=FNULL, stderr=subprocess.STDOUT) except: ll.error("Error when executing post command") delta = datetime.now() - start ll.info("{} completed in {}s".format(task["name"], delta.seconds)) return True except KeyboardInterrupt: print("KeyboardInterrupt caught while processing task {}".format( task["name"])) sys.exit(0) return False except pexpect.exceptions.TIMEOUT: ll.error("Timeout caught") return False except Exception as e: ll.critical("Error while processing {}. continuing".format( task["name"])) ll.critical(str(e)) if ll.getEffectiveLevel() <= logging.DEBUG: traceback.print_exc() return False