def setUp(self):
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.DEBUG)
# Generate series of snapshot names
snapshot_names = list()
now = datetime.now(timezone.utc)
for i in range(0, 24 * 120):
timestamp = now - timedelta(hours=i)
sn = SnapshotName(timestamp=timestamp)
snapshot_names.append(sn)
self.snapshot_names = snapshot_names
def retrieve_snapshots(self):
""" Determine snapshot names. Snapshot names are sorted in reverse order (newest first).
stored internally (self.snapshot_names) and also returned. """
self._log_info('retrieving snapshots')
output = self.exec_check_output('btrfs sub list -o "%s"' %
self.container_subvolume_path)
# output is delivered as a byte sequence, decode to unicode string and split lines
lines = output.decode().splitlines()
subvolumes = list(map(lambda x: Subvolume.parse(x), lines))
# verify snapshot subvolume path consistency
if len(subvolumes) > 0:
subvol_path = os.path.dirname(subvolumes[0].path)
subvol_inconsistent_path = \
next((s.path for s in subvolumes if os.path.dirname(s.path) != subvol_path), None)
if subvol_inconsistent_path:
raise Exception(
'inconsistent path detected at %s [%s != %s], indicating a nested'
' folder/subvolume structure within a container subvolume.'
' each backup job must have a dedicated source/destination container subvolume'
% (self.url.path, subvol_path, subvol_inconsistent_path))
# sort and return
snapshots = []
for sv in subvolumes:
try:
snapshots.append(
Snapshot(SnapshotName.parse(os.path.basename(sv.path)),
sv))
except:
# skip snapshot names which cannot be parsed
pass
self.__snapshots = sorted(snapshots,
key=lambda s: s.name.timestamp,
reverse=True)
return self.__snapshots
def retrieve_snapshots(self):
""" Determine snapshot names. Snapshot names are sorted in reverse order (newest first).
stored internally (self.snapshot_names) and also returned. """
self._log_info('retrieving snapshots')
output = self.exec_check_output('btrfs sub list -o "%s"' % self.container_subvolume_path)
# output is delivered as a byte sequence, decode to unicode string and split lines
lines = output.decode().splitlines()
subvolumes = list(map(lambda x: Subvolume.parse(x), lines))
# verify snapshot subvolume path consistency
if len(subvolumes) > 0:
subvol_path = os.path.dirname(subvolumes[0].path)
subvol_inconsistent_path = \
next((s.path for s in subvolumes if os.path.dirname(s.path) != subvol_path), None)
if subvol_inconsistent_path:
raise Exception('inconsistent path detected at %s [%s != %s], indicating a nested'
' folder/subvolume structure within a container subvolume.'
' each backup job must have a dedicated source/destination container subvolume'
% (self.url.path, subvol_path, subvol_inconsistent_path))
# sort and return
snapshots = []
for sv in subvolumes:
try:
snapshots.append(Snapshot(SnapshotName.parse(os.path.basename(sv.path)), sv))
except:
# skip snapshot names which cannot be parsed
pass
self.__snapshots = sorted(snapshots, key=lambda s: s.name.timestamp, reverse=True)
return self.__snapshots
def test_instantiation(self):
print(SnapshotName.parse('sx-20150102-132010-utc'))
def run(self):
""" Performs backup run """
starting_time = time.monotonic()
_logger.info(self.source)
if self.destination:
_logger.info(self.destination)
# Prepare environments
_logger.info('preparing environment')
self.source.prepare_environment()
if self.destination:
self.destination.prepare_environment()
# Retrieve snapshot names of both source and destination
self.source.retrieve_snapshots()
if self.destination:
self.destination.retrieve_snapshots()
new_snapshot_name = SnapshotName()
if len(self.source.snapshots) > 0 \
and new_snapshot_name.timestamp <= self.source.snapshots[0].name.timestamp:
raise Error(('current snapshot name [%s] would be older than newest existing snapshot [%s] '
'which may indicate a system time problem')
% (new_snapshot_name, self.source.snapshots[0].name))
temp_name = self.source.create_temp_name()
# btrfs send command/subprocess
source_parent_path = None
if len(self.source.snapshots) > 0:
# Indicates if an incremental snapshot can/should be performed
incremental = False
# Latest source and destination snapshot timestamp has to match for incremental transfer
if self.destination is not None:
if len(self.destination.snapshots) > 0:
if self.source.snapshots[0].name.timestamp != self.destination.snapshots[0].name.timestamp:
_logger.warn(
('Latest timestamps of source [%s] and destination [%s] do not match. A full snapshot will '
'be transferred')
% (self.source.snapshots[0].name.timestamp, self.destination.snapshots[0].name.timestamp))
else:
incremental = True
else:
_logger.warn('Destination has no snapshots, a full snapshot will be transferred')
else:
incremental = True
# Set source parent path in case incremental transfer is applicable
if incremental:
source_parent_path = os.path.join(self.source.container_subvolume_path,
str(self.source.snapshots[0].name))
# Create source snapshot
temp_source_path = self.source.create_snapshot(temp_name)
# Recovery handler, swallows all exceptions and logs them
def recover(l, warn_msg: str):
try:
l()
except Exception as e:
_logger.error(str(e))
_logger.warn(warn_msg)
temp_dest_path = None
final_dest_path = None
# Transfer temporary snapshot
if self.destination:
temp_dest_path = self.destination.build_path(temp_name)
final_dest_path = os.path.join(self.destination.url.path, str(new_snapshot_name))
try:
self.source.transfer_btrfs_snapshot(self.destination,
source_path=temp_source_path,
source_parent_path=source_parent_path,
compress=self.source.compress)
except BaseException as e:
recover(lambda: self.source.remove_btrfs_subvolume(temp_source_path),
'could not remove temporary source snapshot [%s]' % temp_source_path)
raise e
try:
final_source_path = os.path.join(self.source.container_subvolume_path, str(new_snapshot_name))
# Rename temporary source snapshot to final snapshot name
self.source.move_file(temp_source_path, final_source_path)
except BaseException as e:
recover(lambda: self.source.remove_btrfs_subvolume(temp_source_path),
'could not remove temporary source snapshot [%s]' % temp_source_path)
if self.destination:
recover(lambda: self.destination.remove_btrfs_subvolume(temp_dest_path),
'could not remove temporary destination snapshot [%s]' % temp_dest_path)
raise e
if self.destination:
try:
# Rename temporary destination snapshot to final snapshot name
self.destination.move_file(temp_dest_path, final_dest_path)
except Exception as e:
# Try to avoid inconsistent state by removing successfully created source snapshot
recover(lambda: self.source.remove_btrfs_subvolume(final_source_path),
'could not remove source snapshot [%s] after failed finalization of destination snapshot'
% final_source_path)
recover(lambda: self.destination.remove_btrfs_subvolume(temp_dest_path),
'could not remove temporary destination snapshot [%s]' % temp_dest_path)
raise e
# Update snapshot name lists
self.source.snapshots.insert(0, Snapshot(new_snapshot_name, None))
if self.destination:
self.destination.snapshots.insert(0, Snapshot(new_snapshot_name, None))
# Clean out excess backups/snapshots
self.source.purge_snapshots()
if self.destination:
self.destination.purge_snapshots()
_logger.info('backup %s created successfully in %s'
% (new_snapshot_name,
time.strftime("%H:%M:%S", time.gmtime(time.monotonic() - starting_time))))
