def test_encrypted_target_but_unencrypted_target_exists():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src", encrypted=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst")
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
properties: false
encryption:
key: password
key-format: passphrase
key-location: $TrueNAS
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
error = run_replication_test(definition, False)
assert "but it already exists and is not encrypted" in error.error
def test_encrypted_target_local():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src")
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst", True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
error = run_replication_test(definition, success=False)
assert "is it's own encryption root" in error.error
def test_zvol_replication__onto_existing_encrypted_unrelated_dataset():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create -V 10M data/src", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True)
create_dataset("data/dst", encrypted=True)
subprocess.check_call("zfs create -V 10M data/dst/vol", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/dev/zvol/data/dst/vol bs=1M count=10", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst/vol
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
error = run_replication_test(definition, success=False)
assert error.error == "Unable to send dataset 'data/src' to existing unrelated encrypted dataset 'data/dst/vol'"
def test_replication_resume__recursive_mount(canmount):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src")
create_dataset("data/src/child")
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs send -R data/src@2018-10-01_01-00 | zfs recv -s -F data/dst", shell=True)
if not canmount:
subprocess.check_call("zfs set canmount=off data/dst", shell=True)
subprocess.check_call("zfs set canmount=off data/dst/child", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("(zfs send -i data/src@2018-10-01_01-00 data/src@2018-10-01_02-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs", shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst",
shell=True, encoding="utf-8")
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
run_replication_test(definition)
mounted = subprocess.check_output("zfs get -H -o value mounted data/dst/child", shell=True, encoding="utf-8")
if canmount:
assert mounted == "yes\n"
else:
assert mounted == "no\n"
def test_name_regex(caplog, transport, all_names, resume):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src")
subprocess.check_call("zfs snapshot -r data/src@snap-2", shell=True)
subprocess.check_call("zfs snapshot -r data/src@manual-1", shell=True)
subprocess.check_call("zfs snapshot -r data/src@snap-1", shell=True)
subprocess.check_call("zfs snapshot -r data/src@manual-2", shell=True)
subprocess.check_call("zfs snapshot -r data/src@snap-3", shell=True)
if resume:
subprocess.check_call(
"zfs send data/src@snap-2 | zfs recv -s -F data/dst", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: false
auto: false
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
if all_names:
definition["replication-tasks"]["src"]["name-regex"] = ".*"
else:
definition["replication-tasks"]["src"]["name-regex"] = "snap-.*"
caplog.set_level(logging.INFO)
run_replication_test(definition)
assert len(list_snapshots(LocalShell(), "data/dst",
False)) == (5 if all_names else 3)
logs = [
record.message for record in caplog.get_records("call")
if "For replication task 'src': doing push" in record.message
]
if all_names:
if resume:
assert len(logs) == 1
else:
assert len(logs) == 2
else:
if resume:
assert len(logs) == 2
else:
assert len(logs) == 3
def test_create_encrypted_target(encryption, key_location, transport):
encryption["key-location"] = key_location
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
if os.path.exists("/tmp/test.key"):
os.unlink("/tmp/test.key")
create_dataset("data/src", encrypted=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
properties: false
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["encryption"] = encryption
definition["replication-tasks"]["src"]["transport"] = transport
run_replication_test(definition)
if key_location == "$TrueNAS":
if encryption["key-format"] != "passphrase":
assert (
subprocess.check_output(["midclt", "call", "-job", "pool.dataset.export_key", "data/dst"]).decode().strip() ==
encryption["key"]
)
else:
assert (
subprocess.check_output("zfs get -H -o value keylocation data/dst", shell=True).decode().strip() ==
f'file://{encryption["key-location"]}'
)
def test_push_replication(compression):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src")
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: ssh
hostname: 127.0.0.1
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
set_localhost_transport_options(
definition["replication-tasks"]["src"]["transport"])
definition["replication-tasks"]["src"]["compression"] = compression
run_replication_test(definition)
def test_encrypted_target(has_data):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
create_dataset("data/src")
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst_parent", True)
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if has_data:
with open("/mnt/data/dst_parent/dst/file", "w"):
pass
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
error = run_replication_test(definition, success=not has_data)
if has_data:
assert "does not have snapshots but has data" in error.error
def test_replication_mount__skip_parent():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
try:
create_dataset("data/src")
create_dataset("data/src/UNIX")
subprocess.check_call("zfs set mountpoint=/UNIX data/src/UNIX",
shell=True)
create_dataset("data/src/UNIX/var")
subprocess.check_call("zfs set mountpoint=/var data/src/UNIX/var",
shell=True)
create_dataset("data/src/UNIX/var/audit")
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
create_dataset("data/dst")
create_dataset("data/dst/server")
create_dataset("data/dst/server/UNIX")
create_dataset("data/dst/server/UNIX/var")
subprocess.check_call(
"zfs set mountpoint=none data/dst/server/UNIX/var", shell=True)
create_dataset("data/dst/server/UNIX/var/audit")
subprocess.check_call(
"zfs set mountpoint=/data/dst/server/var/audit data/dst/server/UNIX/var/audit",
shell=True)
subprocess.check_call("zfs set readonly=on data/dst/server",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src/UNIX
target-dataset: data/dst/server/UNIX
recursive: true
properties: false
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
readonly: set
"""))
run_replication_test(definition)
mounted = subprocess.check_output(
"zfs get -H -o value mounted data/dst/server/UNIX/var/audit",
shell=True,
encoding="utf-8")
assert mounted == "yes\n"
finally:
subprocess.call("zfs destroy -r data/src", shell=True)
def test_replication_resume(caplog, transport, dedup, encrypted):
if dedup and not hasattr(libzfs.SendFlags, "DEDUP"):
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src", encrypted)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
if encrypted:
subprocess.check_call("(zfs send -p -w data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs", shell=True)
else:
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs", shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst",
shell=True, encoding="utf-8")
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["dedup"] = dedup
if encrypted:
definition["replication-tasks"]["src"]["properties"] = True
caplog.set_level(logging.INFO)
run_replication_test(definition)
assert any(
"Resuming replication for destination dataset" in record.message
for record in caplog.get_records("call")
)
local_shell = LocalShell()
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
if not encrypted:
assert subprocess.check_output("zfs get -H -o value mounted data/dst", shell=True, encoding="utf-8") == "yes\n"
def test_replicate_to_existing_dataset_structure(recursive, exclude,
src_has_child,
dst_child_mounted,
dst_child_has_own_contents,
deeply_nested, transport):
if not recursive and exclude:
return
if dst_child_mounted and dst_child_has_own_contents:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
src = "data/src"
create_dataset(src)
if src_has_child:
if deeply_nested:
src = "data/src/deep"
create_dataset(src)
create_dataset(f"{src}/child")
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
dst = "data/dst"
create_dataset(dst)
if deeply_nested:
dst = "data/dst/deep"
create_dataset(dst)
create_dataset(f"{dst}/child")
if not dst_child_mounted:
subprocess.check_call(f"zfs umount {dst}/child", shell=True)
if dst_child_has_own_contents:
with open(f"/mnt/{dst}/child/file", "w") as f:
pass
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: false
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["recursive"] = recursive
if exclude:
definition["replication-tasks"]["src"]["exclude"] = [f"{src}/child"]
definition["replication-tasks"]["src"]["transport"] = transport
if not recursive or exclude or not src_has_child or dst_child_has_own_contents:
error = run_replication_test(definition, success=False)
assert "Refusing to overwrite existing data" in error.error
else:
run_replication_test(definition)
def test_push_replication(dst_parent_is_readonly, dst_exists, transport,
replicate, properties, compression, encrypted,
dst_parent_encrypted):
if transport["type"] != "ssh" and compression:
return
if replicate and not properties:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
create_dataset("data/src", encrypted)
subprocess.check_call("zfs set test:property=test-value data/src",
shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst_parent", dst_parent_encrypted)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["replicate"] = replicate
definition["replication-tasks"]["src"]["properties"] = properties
if compression:
definition["replication-tasks"]["src"]["compression"] = compression
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if dst_exists and properties and encrypted and not dst_parent_encrypted:
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskError), error
assert error.error == (
"Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated "
"dataset 'data/dst_parent/dst'")
return
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (("test-value" in subprocess.check_output(
"zfs get test:property data/dst_parent/dst",
shell=True,
encoding="utf-8")) == properties)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_replication_resume__replicate(caplog, kill_timeout):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
create_dataset("data/src")
create_dataset("data/src/child")
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/child/blob bs=1M count=1",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call(
"(zfs send -R data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
f"sleep {kill_timeout}; killall zfs",
shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output(
"zfs get -H receive_resume_token data/dst",
shell=True,
encoding="utf-8")
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
replicate: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
caplog.set_level(logging.INFO)
run_replication_test(definition)
assert any("Discarding receive_resume_token" in record.message
for record in caplog.get_records("call"))
local_shell = LocalShell()
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/child", False)) == 1
