def test__can_run_in_parallel__different_direction(t1, t2, can):
t1.direction = ReplicationDirection.PUSH
t2.direction = ReplicationDirection.PULL
zettarepl = Zettarepl(Mock(), Mock())
assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def test__can_run_in_parallel__same_direction(t1, t2, can):
with patch("zettarepl.zettarepl.are_same_host", Mock(return_value=True)):
t1.direction = t2.direction = None
zettarepl = Zettarepl(Mock(), Mock())
assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
"""))
set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"])
set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"])
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
