def sync_datasets(self, source_node, source_datasets, target_node,
filter_properties, set_properties):
"""Sync datasets, or thin-only on both sides"""
fail_count = 0
target_datasets = []
for source_dataset in source_datasets:
try:
# determine corresponding target_dataset
target_name = self.args.target_path + "/" + source_dataset.lstrip_path(
self.args.strip_path)
target_dataset = ZfsDataset(target_node, target_name)
target_datasets.append(target_dataset)
# ensure parents exists
# TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't.
if not self.args.no_send \
and target_dataset.parent not in target_datasets \
and not target_dataset.parent.exists:
target_dataset.parent.create_filesystem(parents=True)
# determine common zpool features (cached, so no problem we call it often)
source_features = source_node.get_zfs_pool(
source_dataset.split_path()[0]).features
target_features = target_node.get_zfs_pool(
target_dataset.split_path()[0]).features
common_features = source_features and target_features
# source_dataset.debug("Common features: {}".format(common_features))
source_dataset.sync_snapshots(
target_dataset,
show_progress=self.args.progress,
features=common_features,
filter_properties=filter_properties,
set_properties=set_properties,
ignore_recv_exit_code=self.args.ignore_transfer_errors,
holds=not self.args.no_holds,
rollback=self.args.rollback,
raw=self.args.raw,
also_other_snapshots=self.args.other_snapshots,
no_send=self.args.no_send,
destroy_incompatible=self.args.destroy_incompatible)
except Exception as e:
fail_count = fail_count + 1
source_dataset.error("FAILED: " + str(e))
if self.args.debug:
raise
if not self.args.no_thinning:
self.thin_missing_targets(target_dataset=ZfsDataset(
target_node, self.args.target_path),
used_target_datasets=target_datasets)
if self.args.destroy_missing is not None:
self.destroy_missing_targets(target_dataset=ZfsDataset(
target_node, self.args.target_path),
used_target_datasets=target_datasets)
return fail_count
def selected_datasets(self):
"""determine filesystems that should be backupped by looking at the special autobackup-property, systemwide
returns: list of ZfsDataset
"""
self.debug("Getting selected datasets")
# get all source filesystems that have the backup property
lines = self.run(tab_split=True,
readonly=True,
cmd=[
"zfs", "get", "-t", "volume,filesystem", "-o",
"name,value,source", "-s", "local,inherited",
"-H", "autobackup:" + self.backup_name
])
# determine filesystems that should be actually backupped
selected_filesystems = []
direct_filesystems = []
for line in lines:
(name, value, source) = line
dataset = ZfsDataset(self, name)
if value == "false":
dataset.verbose("Ignored (disabled)")
else:
if source == "local" and (value == "true" or value == "child"):
direct_filesystems.append(name)
if source == "local" and value == "true":
dataset.verbose("Selected (direct selection)")
selected_filesystems.append(dataset)
elif source.find("inherited from ") == 0 and (
value == "true" or value == "child"):
inherited_from = re.sub("^inherited from ", "", source)
if inherited_from in direct_filesystems:
selected_filesystems.append(dataset)
dataset.verbose("Selected (inherited selection)")
else:
dataset.debug("Ignored (already a backup)")
else:
dataset.verbose("Ignored (only childs)")
return selected_filesystems
def selected_datasets(self, ignore_received=True):
"""determine filesystems that should be backupped by looking at the special autobackup-property, systemwide
returns: list of ZfsDataset
"""
self.debug("Getting selected datasets")
# get all source filesystems that have the backup property
lines = self.run(tab_split=True,
readonly=True,
cmd=[
"zfs", "get", "-t", "volume,filesystem", "-o",
"name,value,source", "-H",
"autobackup:" + self.backup_name
])
# The returnlist of selected ZfsDataset's:
selected_filesystems = []
# list of sources, used to resolve inherited sources
sources = {}
for line in lines:
(name, value, raw_source) = line
dataset = ZfsDataset(self, name)
# "resolve" inherited sources
sources[name] = raw_source
if raw_source.find("inherited from ") == 0:
inherited = True
inherited_from = re.sub("^inherited from ", "", raw_source)
source = sources[inherited_from]
else:
inherited = False
source = raw_source
# determine it
if dataset.is_selected(value=value,
source=source,
inherited=inherited,
ignore_received=ignore_received):
selected_filesystems.append(dataset)
return selected_filesystems
def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes):
"""create a consistent (atomic) snapshot of specified datasets, per pool.
"""
pools = {}
# collect snapshots that we want to make, per pool
# self.debug(datasets)
for dataset in datasets:
if not dataset.is_changed_ours(min_changed_bytes):
dataset.verbose("No changes since {}".format(
dataset.our_snapshots[-1].snapshot_name))
continue
# force_exist, since we're making it
snapshot = ZfsDataset(dataset.zfs_node,
dataset.name + "@" + snapshot_name,
force_exists=True)
pool = dataset.split_path()[0]
if pool not in pools:
pools[pool] = []
pools[pool].append(snapshot)
# update cache, but try to prevent an unneeded zfs list
if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'):
dataset.snapshots.append(
snapshot
) # NOTE: this will trigger zfs list if its not cached
if not pools:
self.verbose("No changes anywhere: not creating snapshots.")
return
# create consistent snapshot per pool
for (pool_name, snapshots) in pools.items():
cmd = ["zfs", "snapshot"]
cmd.extend(map(lambda snapshot_: str(snapshot_), snapshots))
self.verbose("Creating snapshots {} in pool {}".format(
snapshot_name, pool_name))
self.run(cmd, readonly=False)
def run(self):
try:
self.verbose(self.HEADER)
if self.args.test:
self.verbose(
"TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES")
self.set_title("Source settings")
description = "[Source]"
if self.args.no_thinning:
source_thinner = None
else:
source_thinner = Thinner(self.args.keep_source)
source_node = ZfsNode(self.args.backup_name,
self,
ssh_config=self.args.ssh_config,
ssh_to=self.args.ssh_source,
readonly=self.args.test,
debug_output=self.args.debug_output,
description=description,
thinner=source_thinner)
source_node.verbose(
"Selects all datasets that have property 'autobackup:{}=true' (or childs of datasets that have "
"'autobackup:{}=child')".format(self.args.backup_name,
self.args.backup_name))
self.set_title("Selecting")
selected_source_datasets = source_node.selected_datasets
if not selected_source_datasets:
self.error(
"No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on the source datasets "
"you want to select.".format(self.args.backup_name))
return 255
# filter out already replicated stuff?
source_datasets = self.filter_replicated(selected_source_datasets)
if not self.args.no_snapshot:
self.set_title("Snapshotting")
source_node.consistent_snapshot(
source_datasets,
source_node.new_snapshotname(),
min_changed_bytes=self.args.min_change)
# if target is specified, we sync the datasets, otherwise we just thin the source. (e.g. snapshot mode)
if self.args.target_path:
# create target_node
self.set_title("Target settings")
if self.args.no_thinning:
target_thinner = None
else:
target_thinner = Thinner(self.args.keep_target)
target_node = ZfsNode(self.args.backup_name,
self,
ssh_config=self.args.ssh_config,
ssh_to=self.args.ssh_target,
readonly=self.args.test,
debug_output=self.args.debug_output,
description="[Target]",
thinner=target_thinner)
target_node.verbose("Receive datasets under: {}".format(
self.args.target_path))
self.set_title("Synchronising")
# check if exists, to prevent vague errors
target_dataset = ZfsDataset(target_node, self.args.target_path)
if not target_dataset.exists:
raise (Exception(
"Target path '{}' does not exist. Please create this dataset first."
.format(target_dataset)))
# do the actual sync
# NOTE: even with no_send, no_thinning and no_snapshot it does a usefull thing because it checks if the common snapshots and shows incompatible snapshots
fail_count = self.sync_datasets(
source_node=source_node,
source_datasets=source_datasets,
target_node=target_node)
#no target specified, run in snapshot-only mode
else:
self.thin_source(source_datasets)
fail_count = 0
if not fail_count:
if self.args.test:
self.set_title("All tests successful.")
else:
self.set_title("All operations completed successfully")
if not self.args.target_path:
self.verbose(
"(No target_path specified, only operated as snapshot tool.)"
)
else:
if fail_count != 255:
self.error("{} failures!".format(fail_count))
if self.args.test:
self.verbose("")
self.verbose("TEST MODE - DID NOT MAKE ANY CHANGES!")
return fail_count
except Exception as e:
self.error("Exception: " + str(e))
if self.args.debug:
raise
return 255
except KeyboardInterrupt:
self.error("Aborted")
return 255
def run(self):
try:
self.verbose(self.HEADER)
if self.args.test:
self.verbose(
"TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES")
self.set_title("Source settings")
description = "[Source]"
source_thinner = Thinner(self.args.keep_source)
source_node = ZfsNode(self.args.backup_name,
self,
ssh_config=self.args.ssh_config,
ssh_to=self.args.ssh_source,
readonly=self.args.test,
debug_output=self.args.debug_output,
description=description,
thinner=source_thinner)
source_node.verbose(
"Selects all datasets that have property 'autobackup:{}=true' (or childs of datasets that have "
"'autobackup:{}=child')".format(self.args.backup_name,
self.args.backup_name))
self.set_title("Selecting")
selected_source_datasets = source_node.selected_datasets
if not selected_source_datasets:
self.error(
"No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on the source datasets "
"you want to select.".format(self.args.backup_name))
return 255
source_datasets = []
# filter out already replicated stuff?
if not self.args.ignore_replicated:
source_datasets = selected_source_datasets
else:
self.set_title("Filtering already replicated filesystems")
for selected_source_dataset in selected_source_datasets:
if selected_source_dataset.is_changed(
self.args.min_change):
source_datasets.append(selected_source_dataset)
else:
selected_source_dataset.verbose(
"Ignoring, already replicated")
if not self.args.no_snapshot:
self.set_title("Snapshotting")
source_node.consistent_snapshot(
source_datasets,
source_node.new_snapshotname(),
min_changed_bytes=self.args.min_change)
# if target is specified, we sync the datasets, otherwise we just thin the source. (e.g. snapshot mode)
if self.args.target_path:
# create target_node
self.set_title("Target settings")
target_thinner = Thinner(self.args.keep_target)
target_node = ZfsNode(self.args.backup_name,
self,
ssh_config=self.args.ssh_config,
ssh_to=self.args.ssh_target,
readonly=self.args.test,
debug_output=self.args.debug_output,
description="[Target]",
thinner=target_thinner)
target_node.verbose("Receive datasets under: {}".format(
self.args.target_path))
# determine filter- and set properties lists
if self.args.filter_properties:
filter_properties = self.args.filter_properties.split(",")
else:
filter_properties = []
if self.args.set_properties:
set_properties = self.args.set_properties.split(",")
else:
set_properties = []
if self.args.clear_refreservation:
filter_properties.append("refreservation")
if self.args.clear_mountpoint:
set_properties.append("canmount=noauto")
if self.args.no_send:
self.set_title("Thinning source and target")
else:
self.set_title("Sending and thinning")
# check if exists, to prevent vague errors
target_dataset = ZfsDataset(target_node, self.args.target_path)
if not target_dataset.exists:
raise (Exception(
"Target path '{}' does not exist. Please create this dataset first."
.format(target_dataset)))
# do the actual sync
fail_count = self.sync_datasets(
source_node=source_node,
source_datasets=source_datasets,
target_node=target_node,
filter_properties=filter_properties,
set_properties=set_properties)
else:
if not self.args.no_thinning:
self.thin_source(source_datasets)
fail_count = 0
if not fail_count:
if self.args.test:
self.set_title("All tests successfull.")
else:
self.set_title("All operations completed successfully")
if not self.args.target_path:
self.verbose(
"(No target_path specified, only operated as snapshot tool.)"
)
else:
if fail_count != 255:
self.error("{} failures!".format(fail_count))
if self.args.test:
self.verbose("")
self.verbose("TEST MODE - DID NOT MAKE ANY CHANGES!")
return fail_count
except Exception as e:
self.error("Exception: " + str(e))
if self.args.debug:
raise
return 255
except KeyboardInterrupt:
self.error("Aborted")
return 255