def test_properties_override(transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: false properties: true properties-override: compression: gzip-9 also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport run_replication_test(definition) assert (subprocess.check_output( "zfs get -H compression data/dst", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "gzip-9")
def test_properties_exclude_override_does_not_break_volume(config): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create -V 1m data/src/vol", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true properties: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"].update(config) run_replication_test(definition)
def test_encrypted_target_replication_from_scratch(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src", True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) create_dataset("data/dst", True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false allow-from-scratch: true retention-policy: none retries: 1 """)) run_replication_test(definition)
def test_creates_intermediate_datasets(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/deeply", shell=True) subprocess.call("zfs destroy -r data/deeply", shell=True) subprocess.check_call("zfs create -V 1M data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/deeply/nested/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) run_replication_test(definition) local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/deeply/nested/dst", False)) == 2
def test_readonly(readonly, dataset_ops): dataset_ops, error = dataset_ops if dataset_ops and readonly == "ignore": return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/sub1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-02_01-00", shell=True) subprocess.check_call("zfs send -R data/src@2018-10-01_01-00 | zfs recv data/dst", shell=True) for dataset, op in dataset_ops: subprocess.check_call(f"zfs set readonly={op} {dataset}", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["readonly"] = readonly if readonly == "require" and error is not None: e = run_replication_test(definition, success=False) assert e.error == error else: run_replication_test(definition) if readonly == "ignore": assert subprocess.check_output( "zfs get -H -o value readonly data/dst/sub1", shell=True, encoding="utf-8" ) == "off\n" else: assert subprocess.check_output( "zfs get -H -o value,source readonly data/dst", shell=True, encoding="utf-8" ) == "on\tlocal\n" assert subprocess.check_output( "zfs get -H -o value,source readonly data/dst/sub1", shell=True, encoding="utf-8" ) == "on\tinherited from data/dst\n"
def test_replication_resume(caplog, transport, dedup): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call( "dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call( "(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output( "zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["dedup"] = dedup caplog.set_level(logging.INFO) run_replication_test(definition) assert any("Resuming replication for destination dataset" in record.message for record in caplog.get_records("call")) local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_name_regex(caplog, transport, all_names, resume): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src") subprocess.check_call("zfs snapshot -r data/src@snap-2", shell=True) subprocess.check_call("zfs snapshot -r data/src@manual-1", shell=True) subprocess.check_call("zfs snapshot -r data/src@snap-1", shell=True) subprocess.check_call("zfs snapshot -r data/src@manual-2", shell=True) subprocess.check_call("zfs snapshot -r data/src@snap-3", shell=True) if resume: subprocess.check_call( "zfs send data/src@snap-2 | zfs recv -s -F data/dst", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: false auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport if all_names: definition["replication-tasks"]["src"]["name-regex"] = ".*" else: definition["replication-tasks"]["src"]["name-regex"] = "snap-.*" caplog.set_level(logging.INFO) run_replication_test(definition) assert len(list_snapshots(LocalShell(), "data/dst", False)) == (5 if all_names else 3) logs = [ record.message for record in caplog.get_records("call") if "For replication task 'src': doing push" in record.message ] if all_names: if resume: assert len(logs) == 1 else: assert len(logs) == 2 else: if resume: assert len(logs) == 2 else: assert len(logs) == 3
def test_replication_resume__recursive_mount(canmount): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src") create_dataset("data/src/child") subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs send -R data/src@2018-10-01_01-00 | zfs recv -s -F data/dst", shell=True) if not canmount: subprocess.check_call("zfs set canmount=off data/dst", shell=True) subprocess.check_call("zfs set canmount=off data/dst/child", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("(zfs send -i data/src@2018-10-01_01-00 data/src@2018-10-01_02-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """)) run_replication_test(definition) mounted = subprocess.check_output("zfs get -H -o value mounted data/dst/child", shell=True, encoding="utf-8") if canmount: assert mounted == "yes\n" else: assert mounted == "no\n"
def test_property_receive(transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) transport["username"] = "******" subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2021-03-10_12-00", shell=True) subprocess.check_call("zfs set truenas:customproperty=1 data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2021-03-10_12-01", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/dst", shell=True) subprocess.check_call("zfs allow user receive,create,mount data/dst/dst", shell=True) subprocess.check_call( "zfs send data/src@2021-03-10_12-00 | zfs recv -s -F data/dst/dst", shell=True) subprocess.check_call("zfs umount data/dst/dst", shell=True) subprocess.check_call("chown user:user /mnt/data/dst/dst", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst/dst recursive: false properties: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 2 """)) definition["replication-tasks"]["src"]["transport"] = transport warning = "cannot receive truenas:customproperty property on data/dst/dst: permission denied" assert warning in run_replication_test(definition).warnings subprocess.check_call("zfs snapshot -r data/src@2021-03-10_12-02", shell=True) assert warning in run_replication_test(definition).warnings
def test_readonly_dst_does_not_exist(readonly): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/sub1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-02_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst/child recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["readonly"] = readonly run_replication_test(definition) if readonly == "ignore": assert subprocess.check_output( "zfs get -H -o value readonly data/dst/child/sub1", shell=True, encoding="utf-8") == "off\n" else: assert subprocess.check_output( "zfs get -H -o value,source readonly data/dst/child", shell=True, encoding="utf-8") == "on\tlocal\n" assert subprocess.check_output( "zfs get -H -o value,source readonly data/dst/child/sub1", shell=True, encoding="utf-8") == "on\tinherited from data/dst/child\n"
def test_nothing_to_replicate(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@manual-snap", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none """)) assert run_replication_test(definition, success=False).error == ( "Dataset 'data/src' does not have any matching snapshots to replicate" )
def test_zvol_replication__onto_existing_dataset(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create -V 1M data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) error = run_replication_test(definition, success=False) assert error.error == "Source 'data/src' is a volume, but target 'data/dst' already exists and is a filesystem"
def test_rewording_is_not_earlier_than_it(transport, direction): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: source-dataset: data/src target-dataset: data/dst recursive: true auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["direction"] = direction definition["replication-tasks"]["src"]["transport"] = transport if direction == "push": definition["replication-tasks"]["src"][ "also-include-naming-schema"] = "%Y-%m-%d_%H-%M" else: definition["replication-tasks"]["src"][ "naming-schema"] = "%Y-%m-%d_%H-%M" error = run_replication_test(definition, False) assert ("is newer than" in error.error and "but has an older date" in error.error)
def test_zvol_replication__onto_existing_encrypted_unrelated_dataset(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create -V 10M data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) create_dataset("data/dst", encrypted=True) subprocess.check_call("zfs create -V 10M data/dst/vol", shell=True) subprocess.check_call("dd if=/dev/urandom of=/dev/zvol/data/dst/vol bs=1M count=10", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst/vol recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) error = run_replication_test(definition, success=False) assert error.error == "Unable to send dataset 'data/src' to existing unrelated encrypted dataset 'data/dst/vol'"
def test_encrypted_target_but_unencrypted_target_exists(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src", encrypted=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) create_dataset("data/dst") definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true properties: false encryption: key: password key-format: passphrase key-location: $TrueNAS also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) error = run_replication_test(definition, False) assert "but it already exists and is not encrypted" in error.error
def test_target_without_snapshots_but_with_data(zvol, mounted, snapdir): if zvol and (not mounted or snapdir): return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) if zvol: subprocess.check_call("zfs create -V 1m data/src", shell=True) else: subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) if zvol: subprocess.check_call("zfs create -V 20m data/dst", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/dev/zvol/data/dst bs=15m count=1", shell=True) else: subprocess.check_call("zfs create data/dst", shell=True) if snapdir: subprocess.check_call("zfs set snapdir=visible data/dst", shell=True) if mounted: bs = "1k" else: bs = "15m" subprocess.check_call( f"dd if=/dev/urandom of=/mnt/data/dst/test bs={bs} count=1", shell=True) if not mounted: subprocess.check_call("zfs unmount data/dst", shell=True) time.sleep(5) # "used" property is not updated immediately definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) error = run_replication_test(definition, success=False) assert "Refusing to overwrite existing data" in error.error
def test_multiple_source_datasets(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/internal", shell=True) subprocess.check_call("zfs create data/src/internal/DISK1", shell=True) subprocess.check_call("zfs create data/src/internal/DISK1/Apps", shell=True) subprocess.check_call("zfs create data/src/internal/DISK1/ISO", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/core", shell=True) subprocess.check_call("zfs send -R data/src/internal/DISK1@2018-10-01_01-00 | " "zfs recv data/dst/core/tsaukpaetra", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: - data/src/internal/DISK1/Apps - data/src/internal/DISK1/ISO target-dataset: data/dst/core/tsaukpaetra recursive: false also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) run_replication_test(definition) local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/dst/core/tsaukpaetra/Apps", False)) == 2 assert len(list_snapshots(local_shell, "data/dst/core/tsaukpaetra/ISO", False)) == 2
def test_create_encrypted_target(encryption, key_location, transport): encryption["key-location"] = key_location subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) if os.path.exists("/tmp/test.key"): os.unlink("/tmp/test.key") create_dataset("data/src", encrypted=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true properties: false also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["encryption"] = encryption definition["replication-tasks"]["src"]["transport"] = transport run_replication_test(definition) if key_location == "$TrueNAS": if encryption["key-format"] != "passphrase": assert ( subprocess.check_output(["midclt", "call", "-job", "pool.dataset.export_key", "data/dst"]).decode().strip() == encryption["key"] ) else: assert ( subprocess.check_output("zfs get -H -o value keylocation data/dst", shell=True).decode().strip() == f'file://{encryption["key-location"]}' )
def test_push_replication(compression): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src") subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: ssh hostname: 127.0.0.1 source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) set_localhost_transport_options( definition["replication-tasks"]["src"]["transport"]) definition["replication-tasks"]["src"]["compression"] = compression run_replication_test(definition)
def test_keeps_mount_structure(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/child", shell=True) subprocess.check_call("zfs create data/src/child/grandchild", shell=True) with open("/mnt/data/src/child/grandchild/file", "w") as f: pass subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) run_replication_test(definition) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) run_replication_test(definition) assert os.path.exists("/mnt/data/dst/child/grandchild/file")
def test_rewording_is_not_earlier_than_it(transport, direction): if transport["type"] == "ssh+netcat": uname = os.uname() if uname.sysname == "FreeBSD" and uname.release.startswith("12"): # FIXME: https://jira.ixsystems.com/browse/NAS-106452 return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: source-dataset: data/src target-dataset: data/dst recursive: true auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["direction"] = direction definition["replication-tasks"]["src"]["transport"] = transport if direction == "push": definition["replication-tasks"]["src"][ "also-include-naming-schema"] = "%Y-%m-%d_%H-%M" else: definition["replication-tasks"]["src"][ "naming-schema"] = "%Y-%m-%d_%H-%M" error = run_replication_test(definition, False) assert ("is newer than" in error.error and "but has an older date" in error.error)
def test_encrypted_target(has_data): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) create_dataset("data/src") subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) create_dataset("data/dst_parent", True) subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if has_data: with open("/mnt/data/dst_parent/dst/file", "w"): pass definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) error = run_replication_test(definition, success=not has_data) if has_data: assert "does not have snapshots but has data" in error.error
def test_readonly_require_zvol(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create -V 1M data/src", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create -V 1M data/dst", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false readonly: require retention-policy: none retries: 1 """)) error = run_replication_test(definition, success=False) assert error.error == ( "Target dataset 'data/dst' exists and does not have readonly=on property, but replication task is set up to " "require this property. Refusing to replicate. Please run \"zfs set readonly=on data/dst\" on the target " "system to fix this.")
def test_preserves_clone_origin(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/iocage", shell=True) subprocess.check_call("zfs create data/src/iocage/child", shell=True) subprocess.check_call("zfs create data/src/iocage/child/dataset", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/iocage/child/dataset/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_14-00", shell=True) subprocess.check_call("zfs create data/src/iocage/another", shell=True) subprocess.check_call("zfs create data/src/iocage/another/child", shell=True) subprocess.check_call( "zfs clone data/src/iocage/child/dataset@2019-11-08_14-00 " "data/src/iocage/another/child/clone", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00", shell=True) assert (subprocess.check_output( "zfs get -H origin data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/src/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6 definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true properties: true replicate: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) run_replication_test(definition) assert (subprocess.check_output( "zfs get -H origin data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/dst/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6
def test_replication_mount__skip_parent(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) try: create_dataset("data/src") create_dataset("data/src/UNIX") subprocess.check_call("zfs set mountpoint=/UNIX data/src/UNIX", shell=True) create_dataset("data/src/UNIX/var") subprocess.check_call("zfs set mountpoint=/var data/src/UNIX/var", shell=True) create_dataset("data/src/UNIX/var/audit") subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) create_dataset("data/dst") create_dataset("data/dst/server") create_dataset("data/dst/server/UNIX") create_dataset("data/dst/server/UNIX/var") subprocess.check_call( "zfs set mountpoint=none data/dst/server/UNIX/var", shell=True) create_dataset("data/dst/server/UNIX/var/audit") subprocess.check_call( "zfs set mountpoint=/data/dst/server/var/audit data/dst/server/UNIX/var/audit", shell=True) subprocess.check_call("zfs set readonly=on data/dst/server", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src/UNIX target-dataset: data/dst/server/UNIX recursive: true properties: false periodic-snapshot-tasks: - src auto: true retention-policy: none readonly: set """)) run_replication_test(definition) mounted = subprocess.check_output( "zfs get -H -o value mounted data/dst/server/UNIX/var/audit", shell=True, encoding="utf-8") assert mounted == "yes\n" finally: subprocess.call("zfs destroy -r data/src", shell=True)
def test_replication_resume__replicate(caplog, kill_timeout): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) create_dataset("data/src") create_dataset("data/src/child") subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/child/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call( "(zfs send -R data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " f"sleep {kill_timeout}; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output( "zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true replicate: true periodic-snapshot-tasks: - src auto: true retention-policy: none """)) caplog.set_level(logging.INFO) run_replication_test(definition) assert any("Discarding receive_resume_token" in record.message for record in caplog.get_records("call")) local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/dst", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/child", False)) == 1
def test_replicate_to_existing_dataset_structure(recursive, exclude, src_has_child, dst_child_mounted, dst_child_has_own_contents, deeply_nested, transport): if not recursive and exclude: return if dst_child_mounted and dst_child_has_own_contents: return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) src = "data/src" create_dataset(src) if src_has_child: if deeply_nested: src = "data/src/deep" create_dataset(src) create_dataset(f"{src}/child") subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) dst = "data/dst" create_dataset(dst) if deeply_nested: dst = "data/dst/deep" create_dataset(dst) create_dataset(f"{dst}/child") if not dst_child_mounted: subprocess.check_call(f"zfs umount {dst}/child", shell=True) if dst_child_has_own_contents: with open(f"/mnt/{dst}/child/file", "w") as f: pass definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: false also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["recursive"] = recursive if exclude: definition["replication-tasks"]["src"]["exclude"] = [f"{src}/child"] definition["replication-tasks"]["src"]["transport"] = transport if not recursive or exclude or not src_has_child or dst_child_has_own_contents: error = run_replication_test(definition, success=False) assert "Refusing to overwrite existing data" in error.error else: run_replication_test(definition)
def test_dst(naming_schemas): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) definition = yaml.safe_load( textwrap.dedent(f"""\ timezone: "Europe/Moscow" periodic-snapshot-tasks: task1: dataset: data/src recursive: true naming-schema: "{naming_schemas[0]}" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" task2: dataset: data/src recursive: true naming-schema: "{naming_schemas[1]}" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" """)) run_periodic_snapshot_test( definition, datetime(2010, 10, 30, 22, 0, 0, tzinfo=pytz.UTC).astimezone(pytz.timezone("Europe/Moscow"))) local_shell = LocalShell() assert list_snapshots(local_shell, "data/src", False) == [ Snapshot("data/src", "auto-2010-10-31-02-00"), Snapshot("data/src", "auto-2010-10-31-02-00:0400"), ] run_periodic_snapshot_test( definition, datetime(2010, 10, 30, 23, 0, 0, tzinfo=pytz.UTC).astimezone(pytz.timezone("Europe/Moscow")), False, ) assert list_snapshots(local_shell, "data/src", False) == [ Snapshot("data/src", "auto-2010-10-31-02-00"), Snapshot("data/src", "auto-2010-10-31-02-00:0300"), Snapshot("data/src", "auto-2010-10-31-02-00:0400"), ] definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "auto-%Y-%m-%d-%H-%M" - "auto-%Y-%m-%d-%H-%M%z" auto: false retention-policy: none retries: 1 """)) run_replication_test(definition) assert list_snapshots(local_shell, "data/dst", False) == [ Snapshot("data/dst", "auto-2010-10-31-02-00"), Snapshot("data/dst", "auto-2010-10-31-02-00:0300"), Snapshot("data/dst", "auto-2010-10-31-02-00:0400"), ]