def test_filter(self):
k = RetentionExpression(
'1d:4/d, 4d:daily, 1w:2/4d, 1m:weekly, 12m:1/y, 23m:none')
#k = KeepExpression('10')
start = time.perf_counter()
(items_to_remove_by_condition,
items_to_keep) = k.filter(self.snapshot_names, lambda x: x.timestamp)
items_to_remove_amount = sum(
map(lambda x: len(x), items_to_remove_by_condition.values()))
items_to_keep_amount = len(items_to_keep)
# for i in items_to_keep:
# print(i)
#
# for c in items_to_remove_by_condition.keys():
# print(c)
# for i in items_to_remove_by_condition[c]:
# print(i)
print('Items to remove %d to keep %d' %
(items_to_remove_amount, items_to_keep_amount))
self.assertEqual(
items_to_remove_amount + items_to_keep_amount,
len(self.snapshot_names),
'Sum of items to keep and remove must be total number of items')
print(time.perf_counter() - start)
def read(self):
cparser = ConfigParser()
if os.path.exists(self.__CONFIG_FILENAME):
with open(self.__CONFIG_FILENAME, 'r') as file:
cparser.read_file(file)
source_retention_str = cparser.get(self.__SECTION_NAME,
self.__KEY_SOURCE_RETENTION,
fallback=None)
dest_retention_str = cparser.get(self.__SECTION_NAME,
self.__KEY_DEST_RETENTION,
fallback=None)
self.__source_retention = RetentionExpression(
source_retention_str) if source_retention_str else None
self.__destination_retention = RetentionExpression(
dest_retention_str) if dest_retention_str else None
self.__log_ident = cparser.get(self.__SECTION_NAME,
self.__KEY_LOG_IDENT,
fallback=None)
self.__email_recipient = cparser.get(self.__SECTION_NAME,
self.__key_EMAIL_RECIPIENT,
fallback=None)
def purge_snapshots(self, retention: RetentionExpression = None):
"""
Purge snapshots
:param retention: Optional override of location's retention
:type retention: RetentionExpression
"""
if retention is None:
retention = self.__retention
else:
self._log_info('Retention expression override [%s]' % retention)
""" Clean out excess backups/snapshots. The newest one (index 0) will always be kept. """
if retention is not None and len(self.__snapshots) > 1:
(to_remove_by_condition,
to_retain) = retention.filter(self.__snapshots[1:],
lambda sn: sn.name.timestamp)
for c in to_remove_by_condition.keys():
to_remove = to_remove_by_condition[c]
self._log_info(
'removing %d snapshot%s due to retention [%s]: %s' %
(len(to_remove), 's' if len(to_remove) > 1 else '', str(c),
', '.join(list(map(lambda x: str(x), to_remove)))))
self.remove_snapshots(list(map(lambda x: str(x), to_remove)))
def init(source_url: parse.SplitResult,
dest_url: parse.SplitResult,
source_container_relpath: str,
source_retention: RetentionExpression = None,
dest_retention: RetentionExpression = None,
compress: bool = None,
identical_filesystem: bool = False) -> 'Job':
"""
Initializes a new backup job
:param source_url: Source url string
:param dest_url: Destination url string
:param source_retention: Source retention expression string
:param dest_retention: Destination retention expression string
:param compress: Compress flag
:return: Backup job
:rtype: Job
"""
source = JobLocation(
source_url,
location_type=JobLocation.TYPE_SOURCE,
container_subvolume_relpath=source_container_relpath)
dest = JobLocation(
dest_url,
location_type=JobLocation.TYPE_DESTINATION) if dest_url else None
if source.has_configuration():
raise Error('source is already initialized')
if dest and dest.has_configuration():
raise Error('destination is already initialized')
# New uuid for both locations
source.uuid = uuid.uuid4()
if dest:
dest.uuid = source.uuid
# Set parameters
if source_retention:
source.retention = source_retention
if not source.retention:
source.retention = Configuration.instance().source_retention
if not source.retention:
source.retention = RetentionExpression(_DEFAULT_RETENTION_SOURCE)
if dest:
if dest_retention:
dest.retention = dest_retention
if not dest.retention:
dest.retention = Configuration.instance().destination_retention
if not dest.retention:
dest.retention = RetentionExpression(
_DEFAULT_RETENTION_DESTINATION)
if compress:
source.compress = compress
if dest:
dest.compress = compress
if not source.compress:
source.compress = False
if dest and not dest.compress:
dest.compress = False
# Check if the filesystem is on identical hosts
if str(source.url.netloc) == str(dest.url.netloc):
_logger.info('Identical hosts for source and destination')
#Check if the filesystems UUID is identical
if source.filesystem == dest.filesystem:
_logger.info('Identical filesystem fo source and destination')
source.identical_filesystem = True
dest.identical_filesystem = True
else:
source.identical_filesystem = False
dest.identical_filesystem = False
_logger.info(
'Source and destination are on different filesystems')
# Prepare environments
_logger.info('preparing source and destination environment')
source.prepare_environment()
if dest:
dest.prepare_environment()
# Writing configurations
source.write_configuration(dest)
if dest:
dest.write_configuration(source)
_logger.info(source)
if dest:
_logger.info(dest)
_logger.info('initialized successfully')
return Job(source, dest)
def read_configuration(self) -> 'JobLocation':
"""
Read configuration file from container subvolume
:return: Corresponding location
"""
# Read configuration file
out = self.exec_check_output('cat "%s"' % self.configuration_filename)
file = out.decode().splitlines()
corresponding_location = None
parser = ConfigParser()
parser.read_file(file)
section = parser.sections()[0]
# Section name implies location type
if section == JobLocation.TYPE_SOURCE:
location_type = JobLocation.TYPE_SOURCE
elif section == JobLocation.TYPE_DESTINATION:
location_type = JobLocation.TYPE_DESTINATION
else:
raise ValueError('invalid section name/location type [%s]' %
section)
# Parse config string values
location_uuid = parser.get(section, self.__KEY_UUID, fallback=None)
source = parser.get(section, self.__KEY_SOURCE, fallback=None)
source_container = parser.get(section,
self.__KEY_SOURCE_CONTAINER,
fallback=None)
destination = parser.get(section,
self.__KEY_DESTINATION,
fallback=None)
# Keep has been renamed to retention.
# Supporting the old name for backward compatibility.
retention = parser.get(section, self.__KEY_RETENTION, fallback=None)
if not retention:
retention = parser.get(section, self.__KEY_KEEP, fallback=None)
# Convert to instances where applicable
location_uuid = UUID(location_uuid) if location_uuid else None
source = parse.urlsplit(source) if source else None
source_container = source_container if source_container else None
destination = parse.urlsplit(destination) if destination else None
retention = RetentionExpression(retention) if retention else None
compress = True if distutils.util.strtobool(parser.get(section, self.__KEY_COMPRESS, fallback='False')) \
else False
identical_filesystem = True if distutils.util.strtobool(parser.get(section, self.__KEY_IDENT_FS, fallback='False')) \
else False
if location_type == JobLocation.TYPE_SOURCE:
# If relpath is not set, this location was initialized from a path to a container subvolume.
# Set source url based on the source read from the configuration.
if not self.container_subvolume_relpath:
self.url = source
self.container_subvolume_relpath = source_container
if destination:
corresponding_location = JobLocation(
destination, location_type=JobLocation.TYPE_DESTINATION)
elif location_type == JobLocation.TYPE_DESTINATION:
if source:
corresponding_location = JobLocation(
source,
location_type=JobLocation.TYPE_SOURCE,
container_subvolume_relpath=source_container)
self.location_type = location_type
self.uuid = location_uuid
self.retention = retention
self.compress = compress
self.identical_filesystem = identical_filesystem
return corresponding_location
import io
import os
import distutils.util
from configparser import ConfigParser
from uuid import UUID
from urllib import parse
from btrfs_pnbackup.entities import Snapshot
from btrfs_pnbackup.entities import SnapshotName
from btrfs_pnbackup.entities import Filesystem
from btrfs_pnbackup.retention import RetentionExpression
from btrfs_pnbackup import shell
from btrfs_pnbackup.entities import Subvolume
_logger = logging.getLogger(__name__)
_DEFAULT_RETENTION_SOURCE = RetentionExpression('3')
_DEFAULT_RETENTION_DESTINATION = RetentionExpression(
'2d: 1/d, 2w:3/w, 1m:1/w, 2m:none')
_DEFAULT_CONTAINER_RELPATH = '.pnbackup'
class Error(Exception):
pass
class Configuration:
""" btrfs-pnbackup global configuration file """
__instance = None
__CONFIG_FILENAME = '/etc/btrfs-pnbackup.conf'
def main():
# Initialize logging
args = parser.parse_args()
# Read global configuration
Configuration.instance().read()
logger = logging.getLogger()
if not args.quiet:
log_std_handler = logging.StreamHandler(sys.stdout)
log_std_handler.setFormatter(
logging.Formatter('%(levelname)s %(message)s'))
logger.addHandler(log_std_handler)
log_memory_handler = None
log_trace = False
email_recipient = None
def handle_exception(ex: Exception):
"""
Exception handler
:param ex:
:return:
"""
# Log exception message
if len(str(ex)) > 0:
logger.error('%s' % str(ex))
if isinstance(ex, CalledProcessError):
if ex.output:
output = ex.output.decode().strip()
if len(output) > 0:
logger.error('%s' % output)
if log_trace:
# Log stack trace
logger.error(traceback.format_exc())
# Email notification
if email_recipient:
try:
# Format message and send
msg = '\n'.join(
map(
lambda log_record: log_memory_handler.formatter.format(
log_record), log_memory_handler.buffer))
mail.send(email_recipient, '%s FAILED' % _APP_NAME, msg)
except Exception as ex:
logger.error(str(ex))
# Syslog handler
if args.command == _CMD_RUN:
log_syslog_handler = logging.handlers.SysLogHandler('/dev/log')
log_syslog_handler.setFormatter(
logging.Formatter(_APP_NAME +
'[%(process)d] %(levelname)s %(message)s'))
logger.addHandler(log_syslog_handler)
# Log ident support
if args.log_ident:
log_ident = args.log_ident if args.log_ident else Configuration.instance(
).log_ident
if log_ident:
log_syslog_handler.ident = log_ident + ' '
# Mail notification support
if args.mail is not None:
email_recipient = args.mail if len(
args.mail) > 0 else Configuration.instance().email_recipient
# Memory handler will buffer output for sending via mail later if needed
log_memory_handler = logging.handlers.MemoryHandler(capacity=-1)
log_memory_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(log_memory_handler)
if args.verbosity and args.verbosity >= 1:
logger.setLevel(logging.DEBUG)
log_trace = True
else:
logger.setLevel(logging.INFO)
logger.info('%s v%s' % (_APP_NAME, __version__))
exitcode = 0
try:
if args.command == _CMD_RUN:
for subvolume in args.subvolumes:
try:
job = Job.load(urllib.parse.urlsplit(subvolume))
job.run()
except Exception as e:
handle_exception(e)
exitcode = 1
elif args.command == _CMD_INIT:
source_retention = RetentionExpression(
args.source_retention) if args.source_retention else None
destination_retention = RetentionExpression(
args.destination_retention
) if args.destination_retention else None
job = Job.init(
source_url=urllib.parse.urlsplit(args.source_subvolume),
source_container_relpath=args.source_container,
source_retention=source_retention,
dest_url=urllib.parse.urlsplit(args.destination_subvolume)
if args.destination_subvolume else None,
dest_retention=destination_retention,
compress=args.compress)
elif args.command == _CMD_UPDATE:
source_retention = RetentionExpression(
args.source_retention) if args.source_retention else None
dest_retention = RetentionExpression(
args.destination_retention
) if args.destination_retention else None
for subvolume in args.subvolumes:
try:
job = Job.load(urllib.parse.urlsplit(subvolume))
job.update(
source_retention=source_retention,
dest_retention=dest_retention,
compress=args.compress if args.compress else
not args.no_compress if args.no_compress else None)
except Exception as e:
handle_exception(e)
exitcode = 1
elif args.command == _CMD_DESTROY:
for subvolume in args.subvolumes:
try:
job = Job.load(urllib.parse.urlsplit(subvolume))
job.destroy(purge=args.purge)
except Exception as e:
handle_exception(e)
exitcode = 1
elif args.command == _CMD_INFO:
for subvolume in args.subvolumes:
try:
job = Job.load(urllib.parse.urlsplit(subvolume),
raise_errors=False)
job.print_info()
except Exception as e:
handle_exception(e)
exitcode = 1
elif args.command == _CMD_PURGE:
source_retention = RetentionExpression(
args.source_retention) if args.source_retention else None
dest_retention = RetentionExpression(
args.destination_retention
) if args.destination_retention else None
for subvolume in args.subvolumes:
try:
job = Job.load(urllib.parse.urlsplit(subvolume))
job.purge(source_retention=source_retention,
dest_retention=dest_retention)
except Exception as e:
handle_exception(e)
exitcode = 1
elif args.command == _CMD_TRANSFER:
source = Location(urllib.parse.urlsplit(args.source_subvolume))
destination = Location(
urllib.parse.urlsplit(args.destination_subvolume))
source.transfer_btrfs_snapshot(destination, compress=args.compress)
except SystemExit as e:
if e.code != 0:
raise
except KeyboardInterrupt as k:
exitcode = 1
except Exception as e:
handle_exception(e)
exitcode = 1
exit(exitcode)