def ensure_has_no_data(shell: Shell, dataset: str):
try:
dst_properties = get_properties(
shell, dataset, {
"type": str,
"mounted": bool,
"mountpoint": str,
"referenced": int,
"snapdir": str,
"used": int,
})
except ExecException as e:
if not ("dataset does not exist" in e.stdout):
raise
else:
if (dst_properties["type"] == "filesystem"
and dst_properties["mounted"]
and dst_properties["mountpoint"] != "legacy"):
try:
index = shell.ls(dst_properties["mountpoint"])
except Exception as e:
logger.warning(
"An exception occurred while listing dataset %r mountpoint %r: %r. Assuming dataset is not mounted",
dataset,
dst_properties["mountpoint"],
e,
)
else:
if dst_properties["snapdir"] == "visible" and ".zfs" in index:
index.remove(".zfs")
if index:
raise ReplicationError(
f"Target dataset {dataset!r} does not have snapshots but has data (e.g. {index[0]!r} and "
f"replication from scratch is not allowed. Refusing to overwrite existing data."
)
return
if dst_properties["type"] == "filesystem":
used_property = "used"
elif dst_properties["type"] == "volume":
used_property = "referenced"
else:
raise ReplicationError(
f"Target dataset {dataset!r} has invalid type {dst_properties['type']!r}"
)
# Empty datasets on large pool configurations can have really big size
if dst_properties[used_property] > 1024 * 1024 * 10:
raise ReplicationError(
f"Target dataset {dataset!r} does not have snapshots but has data ({dst_properties[used_property]} "
f"bytes used) and replication from scratch is not allowed. Refusing to overwrite existing data."
)
def destroy_empty_encrypted_target(replication_task: ReplicationTask,
source_dataset: str,
dst_context: ReplicationContext):
dst_dataset = get_target_dataset(replication_task, source_dataset)
if dst_dataset not in dst_context.datasets:
return
try:
properties = get_properties(dst_context.shell, dst_dataset, {
"encryption": str,
"encryptionroot": str
})
except ExecException as e:
logger.debug(
"Encryption not supported on shell %r: %r (exit code = %d)",
dst_context.shell,
e.stdout.split("\n")[0], e.returncode)
return
if replication_task.encryption and properties["encryption"] == "off":
raise ReplicationError(
f"Encryption requested for destination dataset {dst_dataset!r}, but it already exists "
"and is not encrypted.")
if dst_context.datasets[dst_dataset]:
return
if dst_context.datasets_receive_resume_tokens.get(dst_dataset) is not None:
return
if properties["encryption"] == "off":
return
if properties["encryptionroot"] == dst_dataset:
raise ReplicationError(
f"Destination dataset {dst_dataset!r} already exists and is it's own encryption root. "
"This configuration is not supported yet. If you want to replicate into an encrypted "
"dataset, please, encrypt it's parent dataset.")
try:
index = list_data(dst_context.shell, dst_dataset)
except DatasetIsNotMounted:
logger.debug(
"Encrypted dataset %r is not mounted, not trying to destroy",
dst_dataset)
else:
if not index:
logger.info(
"Encrypted destination dataset %r does not have snapshots or data, destroying it",
dst_dataset)
dst_context.shell.exec(["zfs", "destroy", dst_dataset])
dst_context.datasets.pop(dst_dataset, None)
dst_context.datasets_readonly.pop(dst_dataset, None)
def inspect_data(shell: Shell, dataset: str, exclude: [str]=None):
exclude = exclude or []
try:
dst_properties = get_properties(shell, dataset, {
"type": str,
"mounted": bool,
"mountpoint": str,
"referenced": int,
"snapdir": str,
"used": int,
})
except ExecException as e:
if "dataset does not exist" in e.stdout:
return None, None
raise
else:
if (
dst_properties["type"] == "filesystem" and
dst_properties["mounted"] and
dst_properties["mountpoint"] != "legacy"
):
try:
index = shell.ls(dst_properties["mountpoint"])
except Exception as e:
logger.warning(
"An exception occurred while listing dataset %r mountpoint %r on shell %r: %r. "
"Assuming dataset is not mounted",
dataset, dst_properties["mountpoint"], shell, e,
)
else:
if dst_properties["snapdir"] == "visible" and ".zfs" in index:
index.remove(".zfs")
for excluded in exclude:
if excluded not in index:
continue
child_mountpoint = os.path.join(dst_properties["mountpoint"], excluded)
try:
if not shell.is_dir(child_mountpoint):
continue
except Exception as e:
logger.warning(
"An exception occurred while checking if %r on shell %r is a directory: %r. "
"Assuming it is not",
child_mountpoint, shell, e,
)
continue
child_dataset = os.path.join(dataset, excluded)
try:
child_properties = get_properties(shell, child_dataset, {
"type": str,
"mounted": bool,
"mountpoint": str,
})
except Exception as e:
logger.warning(
"An exception occurred while getting properties for dataset %r on shell %r: %r. "
"Assuming it does not exist",
child_dataset, shell, e,
)
continue
if child_properties["type"] == "filesystem":
if child_properties["mounted"] and child_properties["mountpoint"] == child_mountpoint:
index.remove(excluded)
else:
try:
child_contents = shell.ls(child_mountpoint)
except Exception as e:
logger.warning(
"An exception occurred while listing %r on shell %r: %r. Assuming it is not empty",
child_mountpoint, shell, e,
)
continue
else:
if not child_contents:
index.remove(excluded)
return index, dst_properties
return None, dst_properties
def calculate_replication_step_templates(replication_task: ReplicationTask,
source_dataset: str,
src_context: ReplicationContext,
dst_context: ReplicationContext):
src_context.datasets = list_datasets_with_snapshots(
src_context.shell, source_dataset, replication_task.recursive)
if replication_task.properties:
src_context.datasets_encrypted = get_datasets_encrypted(
src_context.shell, source_dataset, replication_task.recursive)
# It's not fail-safe to send recursive streams because recursive snapshots can have excludes in the past
# or deleted empty snapshots
source_datasets = src_context.datasets.keys(
) # Order is right because it's OrderedDict
if replication_task.replicate:
# But when replicate is on, we have no choice
source_datasets = [source_dataset]
dst_context.datasets = {}
dst_context.datasets_readonly = {}
dst_context.datasets_receive_resume_tokens = {}
templates = []
for source_dataset in source_datasets:
if not replication_task_should_replicate_dataset(
replication_task, source_dataset):
continue
target_dataset = get_target_dataset(replication_task, source_dataset)
valid_properties = set()
referenced_properties = (
set(replication_task.properties_exclude)
| set(replication_task.properties_override.keys()))
if referenced_properties:
for property, (value, source) in get_properties(
src_context.shell, source_dataset,
{p: str
for p in referenced_properties}, True).items():
if source == "-":
# Invalid properties (like `mountpoint` for VOLUME) will be seen like this
# Properties like `used` will be seen like this too, but they will never appear in
# `referenced_properties` because no one excludes or overrides them.
continue
valid_properties.add(property)
try:
datasets = list_datasets_with_properties(
dst_context.shell, target_dataset, replication_task.recursive,
{
"readonly": str,
"receive_resume_token": str
})
except DatasetDoesNotExistException:
pass
else:
dst_context.datasets.update(
list_snapshots_for_datasets(
dst_context.shell, target_dataset,
replication_task.recursive,
[dataset["name"] for dataset in datasets]))
dst_context.datasets_readonly.update(
**{
dataset["name"]: zfs_bool(dataset["readonly"])
for dataset in datasets
})
dst_context.datasets_receive_resume_tokens.update(
**{
dataset["name"]: dataset["receive_resume_token"]
if dataset["receive_resume_token"] != "-" else None
for dataset in datasets
})
templates.append(
ReplicationStepTemplate(replication_task, src_context, dst_context,
source_dataset, target_dataset,
valid_properties))
return templates