def test_push_replication(transport, properties):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst", shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def create_zettarepl(definition):
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock(return_value=None)
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
return zettarepl
def test_replication_resume(caplog, transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs", shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst",
shell=True, encoding="utf-8")
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition = Definition.from_data(definition)
caplog.set_level(logging.INFO)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert any(
"Resuming replication for destination dataset" in record.message
for record in caplog.get_records("call")
)
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_push_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = Definition.from_data(yaml.load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def test_zvol_replication(as_root):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
if as_root:
subprocess.check_call("zfs create -V 1M data/src", shell=True)
else:
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create -V 1M data/src/zvol", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
if not as_root:
assert len(list_snapshots(local_shell, "data/dst/zvol", False)) == 2
def test_push_replication(dst_parent_is_readonly, dst_exists, transport,
properties, compression):
if transport["type"] != "ssh" and compression:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src",
shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst_parent", shell=True)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
if compression:
definition["replication-tasks"]["src"]["compression"] = compression
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (("test-value" in subprocess.check_output(
"zfs get test:property data/dst_parent/dst",
shell=True,
encoding="utf-8")) == properties)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_snapshot_gone(transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 2
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
deleted = False
def resume_replications_mock(*args, **kwargs):
nonlocal deleted
if not deleted:
# Snapshots are already listed, and now we remove one of them to simulate PULL replication
# from remote system that has `allow_empty_snapshots: false`. Only do this once.
subprocess.check_call("zfs destroy data/src@2018-10-01_01-00",
shell=True)
deleted = True
return resume_replications(*args, **kwargs)
with patch("zettarepl.replication.run.resume_replications",
resume_replications_mock):
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
"""))
set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"])
set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"])
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, replicate, properties, encrypted,
has_encrypted_child, dst_parent_encrypted):
if replicate and not properties:
return
if encrypted and has_encrypted_child:
# If parent is encrypted, child is also encrypted
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
create_dataset("data/src", encrypted)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
if has_encrypted_child:
create_dataset("data/src/child", True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst_parent", dst_parent_encrypted)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["replicate"] = replicate
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if dst_exists and properties and encrypted and not dst_parent_encrypted:
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskError), error
assert error.error == ("Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated "
"dataset 'data/dst_parent/dst'")
return
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst_parent/dst",
shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_preserves_clone_origin():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/iocage", shell=True)
subprocess.check_call("zfs create data/src/iocage/child", shell=True)
subprocess.check_call("zfs create data/src/iocage/child/dataset",
shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/iocage/child/dataset/blob bs=1M count=1",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2019-11-08_14-00",
shell=True)
subprocess.check_call("zfs create data/src/iocage/another", shell=True)
subprocess.check_call("zfs create data/src/iocage/another/child",
shell=True)
subprocess.check_call(
"zfs clone data/src/iocage/child/dataset@2019-11-08_14-00 "
"data/src/iocage/another/child/clone",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00",
shell=True)
assert (subprocess.check_output(
"zfs get -H origin data/src/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2] ==
"data/src/iocage/child/dataset@2019-11-08_14-00")
assert int(
subprocess.check_output(
"zfs get -H -p used data/src/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2]) < 2e6
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
properties: true
replicate: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert (subprocess.check_output(
"zfs get -H origin data/dst/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2] ==
"data/dst/iocage/child/dataset@2019-11-08_14-00")
assert int(
subprocess.check_output(
"zfs get -H -p used data/dst/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2]) < 2e6
