async def get_definition(self):
config = await self.middleware.call("replication.config.config")
timezone = (await self.middleware.call("system.general.config"))["timezone"]
pools = {pool["name"]: pool for pool in await self.middleware.call("pool.query")}
hold_tasks = {}
periodic_snapshot_tasks = {}
for periodic_snapshot_task in await self.middleware.call("pool.snapshottask.query", [["enabled", "=", True]]):
hold_task_reason = self._hold_task_reason(pools, periodic_snapshot_task["dataset"])
if hold_task_reason:
hold_tasks[f"periodic_snapshot_task_{periodic_snapshot_task['id']}"] = hold_task_reason
continue
periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = self.periodic_snapshot_task_definition(
periodic_snapshot_task,
)
replication_tasks = {}
for replication_task in await self.middleware.call("replication.query", [["enabled", "=", True]]):
try:
replication_tasks[f"task_{replication_task['id']}"] = await self._replication_task_definition(
pools, replication_task
)
except HoldReplicationTaskException as e:
hold_tasks[f"replication_task_{replication_task['id']}"] = e.reason
for job_id, replication_task in self.onetime_replication_tasks.items():
try:
replication_tasks[f"job_{job_id}"] = await self._replication_task_definition(pools, replication_task)
except HoldReplicationTaskException as e:
hold_tasks[f"job_{job_id}"] = e.reason
definition = {
"max-parallel-replication-tasks": config["max_parallel_replication_tasks"],
"timezone": timezone,
"use-removal-dates": True,
"periodic-snapshot-tasks": periodic_snapshot_tasks,
"replication-tasks": replication_tasks,
}
# Test if does not cause exceptions
Definition.from_data(definition, raise_on_error=False)
hold_tasks = {
task_id: {
"state": "HOLD",
"datetime": datetime.utcnow(),
"reason": make_sentence(reason),
}
for task_id, reason in hold_tasks.items()
}
return definition, hold_tasks
def test_push_replication(transport, properties):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst", shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "config":
if "max_parallel_replication_tasks" in args:
self.zettarepl.max_parallel_replication_tasks = args[
"max_parallel_replication_tasks"]
if "timezone" in args:
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args["timezone"])
if command == "tasks":
definition = Definition.from_data(args, raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
self.zettarepl.set_tasks(definition.tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
self.observer_queue.put(
ReplicationTaskError(task_id, "Task not found"))
def __call__(self):
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
debug_level = "DEBUG"
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
debug_level = "INFO"
else:
debug_level = self.debug_level
setup_logging("zettarepl", debug_level, self.log_handler)
definition = Definition.from_data(self.definition)
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def test_does_not_remove_the_last_snapshot_left():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2020-05-07_00-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2020-05-23_00-00", shell=True)
data = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: false
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "*"
hour: "*"
day-of-month: "*"
month: "*"
day-of-week: "*"
lifetime: P30D
"""))
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2020, 6, 25, 0, 0))
assert list_snapshots(local_shell, "data/src",
False) == [Snapshot("data/src", "2020-05-23_00-00")]
def test_push_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/child", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert sum(1 for m in zettarepl.observer.call_args_list
if isinstance(m[0][0], ReplicationTaskSuccess)) == 1
subprocess.check_call("zfs destroy -r data/src/child", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert sum(1 for m in zettarepl.observer.call_args_list
if isinstance(m[0][0], ReplicationTaskSuccess)) == 2
local_shell = LocalShell()
assert len(list_snapshots(local_shell, "data/dst/child", False)) == 1
def test_replication_resume(caplog):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call(
"dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call(
"(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs",
shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output(
"zfs get -H receive_resume_token data/dst",
shell=True,
encoding="utf-8")
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
- id: src
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
- id: src
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
assert any("Resuming replication for dst_dataset" in record.message
for record in caplog.get_records("call"))
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def run_periodic_snapshot_test(definition, now, success=True):
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._run_periodic_snapshot_tasks(now, select_by_class(PeriodicSnapshotTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if success:
for call in zettarepl.observer.call_args_list:
call = call[0][0]
assert not isinstance(call, PeriodicSnapshotTaskError), success
def test_replication_data_progress():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: ssh
hostname: 127.0.0.1
source-dataset:
- data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
set_localhost_transport_options(
definition["replication-tasks"]["src"]["transport"])
definition["replication-tasks"]["src"]["speed-limit"] = 10240 * 9
with patch("zettarepl.replication.run.DatasetSizeObserver.INTERVAL", 5):
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
calls = [
call for call in zettarepl.observer.call_args_list
if call[0][0].__class__ == ReplicationTaskDataProgress
]
assert len(calls) == 2
assert 1024 * 1024 * 0.8 <= calls[0][0][0].src_size <= 1024 * 1024 * 1.2
assert 0 <= calls[0][0][0].dst_size <= 10240 * 1.2
assert 1024 * 1024 * 0.8 <= calls[1][0][0].src_size <= 1024 * 1024 * 1.2
assert 10240 * 6 * 0.8 <= calls[1][0][0].dst_size <= 10240 * 6 * 1.2
def test_push_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = Definition.from_data(yaml.load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def run_replication_test(definition, success=True):
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if success:
success = zettarepl.observer.call_args_list[-1][0][0]
assert isinstance(success, ReplicationTaskSuccess), success
else:
error = zettarepl.observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskError), error
return error
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
osc.die_with_parent()
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
default_level = logging.DEBUG
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
default_level = logging.INFO
else:
default_level = logging.getLevelName(self.debug_level)
setup_logging("", "DEBUG", self.log_handler)
oqlh = ObserverQueueLoggingHandler(self.observer_queue)
oqlh.setFormatter(
logging.Formatter(
'[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s',
'%Y/%m/%d %H:%M:%S'))
logging.getLogger("zettarepl").addHandler(oqlh)
for handler in logging.getLogger("zettarepl").handlers:
handler.addFilter(LongStringsFilter())
handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level))
c = Client('ws+unix:///var/run/middlewared-internal.sock',
py_exceptions=True)
c.subscribe(
'core.reconfigure_logging',
lambda *args, **kwargs: reconfigure_logging('zettarepl_file'))
definition = Definition.from_data(self.definition,
raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def test_source_retention_multiple_sources():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_02-00", shell=True)
definition = Definition.from_data(yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: [data/src/a, data/src/b]
target-dataset: data/dst
recursive: false
periodic-snapshot-tasks:
- src
auto: true
retention-policy: source
hold-pending-snapshots: true
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst/a", False) == [Snapshot("data/dst/a", "2018-10-01_02-00")]
assert list_snapshots(local_shell, "data/dst/b", False) == [Snapshot("data/dst/b", "2018-10-01_02-00")]
def test_push_remote_retention(retention_policy, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True)
data = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
"""))
data["replication-tasks"]["src"].update(**retention_policy)
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst", False) == remains
def test_hold_pending_snapshots(hold_pending_snapshots, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
- id: src
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
- id: src
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: source
hold-pending-snapshots: """ + yaml.dump(hold_pending_snapshots) +
"""
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/src", False) == remains
def test_zvol_replication(as_root):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
if as_root:
subprocess.check_call("zfs create -V 1M data/src", shell=True)
else:
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create -V 1M data/src/zvol", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
if not as_root:
assert len(list_snapshots(local_shell, "data/dst/zvol", False)) == 2
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(args)
if command == "tasks":
self.zettarepl.set_tasks(Definition.from_data(args).tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name, task_id)
def test_hold_pending_snapshots(retention_policy, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True)
data = yaml.load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
- id: src
direction: pull
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
naming-schema: "%Y-%m-%d_%H-%M"
recursive: true
auto: true
"""))
data["replication-tasks"][0].update(**retention_policy)
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst", False) == remains
def test_does_not_remove_the_last_snapshot_left(snapshots__removal_dates__result):
snapshots, removal_dates, result = snapshots__removal_dates__result
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/src2", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/child", shell=True)
subprocess.check_call("zfs create data/src2", shell=True)
for snapshot in snapshots:
subprocess.check_call(f"zfs snapshot {snapshot}", shell=True)
data = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: false
naming-schema: "%Y-%m-%d-%H-%M"
schedule:
minute: "*"
hour: "*"
day-of-month: "*"
month: "*"
day-of-week: "*"
lifetime: P30D
"""))
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell, use_removal_dates=True)
zettarepl.set_tasks(definition.tasks)
with patch("zettarepl.zettarepl.get_removal_dates", Mock(return_value=removal_dates)):
zettarepl._run_local_retention(datetime(2021, 4, 19, 17, 0))
assert list_snapshots(local_shell, "data/src", False) + list_snapshots(local_shell, "data/src2", False) == [
snapshots[i] for i in result
]
def run(args):
try:
definition = Definition.from_data(yaml.load(args.definition_path))
except yaml.YAMLError as e:
sys.stderr.write(f"Definition syntax error: {e!s}\n")
sys.exit(1)
except jsonschema.exceptions.ValidationError as e:
sys.stderr.write(f"Definition validation error: {e!s}\n")
sys.exit(1)
except ValueError as e:
sys.stderr.write(f"{e!s}\n")
sys.exit(1)
clock = Clock(args.once)
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
zettarepl = Zettarepl(scheduler, local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl.run()
def test_pull_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
- id: src
direction: pull
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
naming-schema:
- "%Y-%m-%d_%H-%M"
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
def test_replication_progress_resume():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_04-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call(
"zfs send data/src@2018-10-01_01-00 | zfs recv -s -F data/dst",
shell=True)
subprocess.check_call(
"(zfs send -i data/src@2018-10-01_01-00 data/src@2018-10-01_02-00 | "
" throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs",
shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output(
"zfs get -H receive_resume_token data/dst",
shell=True,
encoding="utf-8")
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
calls = [
call for call in zettarepl.observer.call_args_list
if call[0][0].__class__ != ReplicationTaskDataProgress
]
result = [
ReplicationTaskStart("src"),
ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_02-00", 0,
3),
ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_02-00",
1, 3),
ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_03-00", 1,
3),
ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_03-00",
2, 3),
ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_04-00", 2,
3),
ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_04-00",
3, 3),
ReplicationTaskSuccess("src"),
]
for i, message in enumerate(result):
call = calls[i]
assert call[0][0].__class__ == message.__class__
d1 = call[0][0].__dict__
d2 = message.__dict__
assert d1 == d2
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
"""))
set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"])
set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"])
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 2
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 4
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
async def get_definition(self):
timezone = (await
self.middleware.call("system.general.config"))["timezone"]
pools = {
pool["name"]: pool
for pool in await self.middleware.call("pool.query")
}
hold_tasks = {}
periodic_snapshot_tasks = {}
for periodic_snapshot_task in await self.middleware.call(
"pool.snapshottask.query", [["enabled", "=", True]]):
hold_task_reason = self._hold_task_reason(
pools, periodic_snapshot_task["dataset"])
if hold_task_reason:
hold_tasks[
f"periodic_snapshot_task_{periodic_snapshot_task['id']}"] = hold_task_reason
continue
periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = {
"dataset":
periodic_snapshot_task["dataset"],
"recursive":
periodic_snapshot_task["recursive"],
"exclude":
periodic_snapshot_task["exclude"],
"lifetime":
lifetime_iso8601(periodic_snapshot_task["lifetime_value"],
periodic_snapshot_task["lifetime_unit"]),
"naming-schema":
periodic_snapshot_task["naming_schema"],
"schedule":
zettarepl_schedule(periodic_snapshot_task["schedule"]),
"allow-empty":
periodic_snapshot_task["allow_empty"],
}
replication_tasks = {}
for replication_task in await self.middleware.call(
"replication.query", [["enabled", "=", True]]):
if replication_task["direction"] == "PUSH":
hold = False
for source_dataset in replication_task["source_datasets"]:
hold_task_reason = self._hold_task_reason(
pools, source_dataset)
if hold_task_reason:
hold_tasks[
f"replication_task_{replication_task['id']}"] = hold_task_reason
hold = True
break
if hold:
continue
if replication_task["direction"] == "PULL":
hold_task_reason = self._hold_task_reason(
pools, replication_task["target_dataset"])
if hold_task_reason:
hold_tasks[
f"replication_task_{replication_task['id']}"] = hold_task_reason
continue
if replication_task["transport"] != "LOCAL":
if not await self.middleware.call(
"network.general.can_perform_activity", "replication"):
hold_tasks[
f"replication_task_{replication_task['id']}"] = (
"Replication network activity is disabled")
continue
try:
transport = await self._define_transport(
replication_task["transport"],
(replication_task["ssh_credentials"] or {}).get("id"),
replication_task["netcat_active_side"],
replication_task["netcat_active_side_listen_address"],
replication_task["netcat_active_side_port_min"],
replication_task["netcat_active_side_port_max"],
replication_task["netcat_passive_side_connect_address"],
)
except CallError as e:
hold_tasks[
f"replication_task_{replication_task['id']}"] = e.errmsg
continue
my_periodic_snapshot_tasks = [
f"task_{periodic_snapshot_task['id']}"
for periodic_snapshot_task in
replication_task["periodic_snapshot_tasks"]
]
my_schedule = replication_task["schedule"]
definition = {
"direction":
replication_task["direction"].lower(),
"transport":
transport,
"source-dataset":
replication_task["source_datasets"],
"target-dataset":
replication_task["target_dataset"],
"recursive":
replication_task["recursive"],
"exclude":
replication_task_exclude(replication_task),
"properties":
replication_task["properties"],
"properties-exclude":
replication_task["properties_exclude"],
"properties-override":
replication_task["properties_override"],
"replicate":
replication_task["replicate"],
"periodic-snapshot-tasks":
my_periodic_snapshot_tasks,
"auto":
replication_task["auto"],
"only-matching-schedule":
replication_task["only_matching_schedule"],
"allow-from-scratch":
replication_task["allow_from_scratch"],
"readonly":
replication_task["readonly"].lower(),
"hold-pending-snapshots":
replication_task["hold_pending_snapshots"],
"retention-policy":
replication_task["retention_policy"].lower(),
"large-block":
replication_task["large_block"],
"embed":
replication_task["embed"],
"compressed":
replication_task["compressed"],
"retries":
replication_task["retries"],
"logging-level": (replication_task["logging_level"]
or "NOTSET").lower(),
}
if replication_task["encryption"]:
definition["encryption"] = {
"key": replication_task["encryption_key"],
"key-format": replication_task["encryption_key_format"],
"key-location":
replication_task["encryption_key_location"],
}
if replication_task["naming_schema"]:
definition["naming-schema"] = replication_task["naming_schema"]
if replication_task["also_include_naming_schema"]:
definition["also-include-naming-schema"] = replication_task[
"also_include_naming_schema"]
if my_schedule is not None:
definition["schedule"] = zettarepl_schedule(my_schedule)
if replication_task["restrict_schedule"] is not None:
definition["restrict-schedule"] = zettarepl_schedule(
replication_task["restrict_schedule"])
if replication_task[
"lifetime_value"] is not None and replication_task[
"lifetime_unit"] is not None:
definition["lifetime"] = lifetime_iso8601(
replication_task["lifetime_value"],
replication_task["lifetime_unit"])
if replication_task["compression"] is not None:
definition["compression"] = replication_task[
"compression"].lower()
if replication_task["speed_limit"] is not None:
definition["speed-limit"] = replication_task["speed_limit"]
replication_tasks[f"task_{replication_task['id']}"] = definition
definition = {
"timezone": timezone,
"periodic-snapshot-tasks": periodic_snapshot_tasks,
"replication-tasks": replication_tasks,
}
# Test if does not cause exceptions
Definition.from_data(definition, raise_on_error=False)
hold_tasks = {
task_id: {
"state": "HOLD",
"datetime": datetime.utcnow(),
"reason": make_sentence(reason),
}
for task_id, reason in hold_tasks.items()
}
return definition, hold_tasks
async def get_definition(self):
timezone = (await
self.middleware.call("system.general.config"))["timezone"]
periodic_snapshot_tasks = {}
for periodic_snapshot_task in await self.middleware.call(
"pool.snapshottask.query",
[["enabled", "=", True], ["legacy", "=", False]]):
periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = {
"dataset":
periodic_snapshot_task["dataset"],
"recursive":
periodic_snapshot_task["recursive"],
"exclude":
periodic_snapshot_task["exclude"],
"lifetime":
lifetime_iso8601(periodic_snapshot_task["lifetime_value"],
periodic_snapshot_task["lifetime_unit"]),
"naming-schema":
periodic_snapshot_task["naming_schema"],
"schedule":
zettarepl_schedule(periodic_snapshot_task["schedule"]),
"allow-empty":
periodic_snapshot_task["allow_empty"],
}
replication_tasks = {}
legacy_periodic_snapshot_tasks_ids = {
periodic_snapshot_task["id"]
for periodic_snapshot_task in await self.middleware.call(
"pool.snapshottask.query", [["legacy", "=", True]])
}
for replication_task in await self.middleware.call(
"replication.query",
[["transport", "!=", "LEGACY"], ["enabled", "=", True]]):
my_periodic_snapshot_tasks = [
f"task_{periodic_snapshot_task['id']}"
for periodic_snapshot_task in
replication_task["periodic_snapshot_tasks"]
if periodic_snapshot_task["id"] not in
legacy_periodic_snapshot_tasks_ids
]
my_schedule = replication_task["schedule"]
# All my periodic snapshot tasks are legacy
if (replication_task["direction"] == "PUSH"
and replication_task["auto"]
and replication_task["periodic_snapshot_tasks"]
and not my_periodic_snapshot_tasks):
my_schedule = replication_task["periodic_snapshot_tasks"][0][
"schedule"]
definition = {
"direction":
replication_task["direction"].lower(),
"transport":
await self._define_transport(
replication_task["transport"],
(replication_task["ssh_credentials"] or {}).get("id"),
replication_task["netcat_active_side"],
replication_task["netcat_active_side_listen_address"],
replication_task["netcat_active_side_port_min"],
replication_task["netcat_active_side_port_max"],
replication_task["netcat_passive_side_connect_address"],
),
"source-dataset":
replication_task["source_datasets"],
"target-dataset":
replication_task["target_dataset"],
"recursive":
replication_task["recursive"],
"exclude":
replication_task_exclude(replication_task["source_datasets"],
replication_task["recursive"],
replication_task["exclude"]),
"properties":
replication_task["properties"],
"periodic-snapshot-tasks":
my_periodic_snapshot_tasks,
"auto":
replication_task["auto"],
"only-matching-schedule":
replication_task["only_matching_schedule"],
"allow-from-scratch":
replication_task["allow_from_scratch"],
"hold-pending-snapshots":
replication_task["hold_pending_snapshots"],
"retention-policy":
replication_task["retention_policy"].lower(),
"dedup":
replication_task["dedup"],
"large-block":
replication_task["large_block"],
"embed":
replication_task["embed"],
"compressed":
replication_task["compressed"],
"retries":
replication_task["retries"],
"logging-level": (replication_task["logging_level"]
or "NOTSET").lower(),
}
if replication_task["naming_schema"]:
definition["naming-schema"] = replication_task["naming_schema"]
if replication_task["also_include_naming_schema"]:
definition["also-include-naming-schema"] = replication_task[
"also_include_naming_schema"]
# Use snapshots created by legacy periodic snapshot tasks
for periodic_snapshot_task in replication_task[
"periodic_snapshot_tasks"]:
if periodic_snapshot_task[
"id"] in legacy_periodic_snapshot_tasks_ids:
definition.setdefault("also-include-naming-schema", [])
definition["also-include-naming-schema"].append(
periodic_snapshot_task["naming_schema"])
if my_schedule is not None:
definition["schedule"] = zettarepl_schedule(my_schedule)
if replication_task["restrict_schedule"] is not None:
definition["restrict-schedule"] = zettarepl_schedule(
replication_task["restrict_schedule"])
if replication_task[
"lifetime_value"] is not None and replication_task[
"lifetime_unit"] is not None:
definition["lifetime"] = lifetime_iso8601(
replication_task["lifetime_value"],
replication_task["lifetime_unit"])
if replication_task["compression"] is not None:
definition["compression"] = replication_task["compression"]
if replication_task["speed_limit"] is not None:
definition["speed-limit"] = replication_task["speed_limit"]
replication_tasks[f"task_{replication_task['id']}"] = definition
definition = {
"timezone": timezone,
"periodic-snapshot-tasks": periodic_snapshot_tasks,
"replication-tasks": replication_tasks,
}
# Test if does not cause exceptions
Definition.from_data(definition, raise_on_error=False)
return definition
def test_push_replication(dst_parent_is_readonly, dst_exists, transport,
properties, compression):
if transport["type"] != "ssh" and compression:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src",
shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst_parent", shell=True)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
if compression:
definition["replication-tasks"]["src"]["compression"] = compression
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (("test-value" in subprocess.check_output(
"zfs get test:property data/dst_parent/dst",
shell=True,
encoding="utf-8")) == properties)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_replication_progress(transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/src1", shell=True)
subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_01-00",
shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/src1/blob bs=1M count=1",
shell=True)
subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_02-00",
shell=True)
subprocess.check_call("rm /mnt/data/src/src1/blob", shell=True)
subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_03-00",
shell=True)
subprocess.check_call("zfs create data/src/src2", shell=True)
subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_02-00",
shell=True)
subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_03-00",
shell=True)
subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_04-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
source-dataset:
- data/src/src1
- data/src/src2
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
if transport["type"] == "ssh":
definition["replication-tasks"]["src"]["speed-limit"] = 10240 * 9
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
calls = [
call for call in zettarepl.observer.call_args_list
if call[0][0].__class__ != ReplicationTaskDataProgress
]
result = [
ReplicationTaskStart("src"),
ReplicationTaskSnapshotStart("src", "data/src/src1",
"2018-10-01_01-00", 0, 3),
ReplicationTaskSnapshotSuccess("src", "data/src/src1",
"2018-10-01_01-00", 1, 3),
ReplicationTaskSnapshotStart("src", "data/src/src1",
"2018-10-01_02-00", 1, 3),
ReplicationTaskSnapshotSuccess("src", "data/src/src1",
"2018-10-01_02-00", 2, 3),
ReplicationTaskSnapshotStart("src", "data/src/src1",
"2018-10-01_03-00", 2, 3),
ReplicationTaskSnapshotSuccess("src", "data/src/src1",
"2018-10-01_03-00", 3, 3),
ReplicationTaskSnapshotStart("src", "data/src/src2",
"2018-10-01_01-00", 3, 7),
ReplicationTaskSnapshotSuccess("src", "data/src/src2",
"2018-10-01_01-00", 4, 7),
ReplicationTaskSnapshotStart("src", "data/src/src2",
"2018-10-01_02-00", 4, 7),
ReplicationTaskSnapshotSuccess("src", "data/src/src2",
"2018-10-01_02-00", 5, 7),
ReplicationTaskSnapshotStart("src", "data/src/src2",
"2018-10-01_03-00", 5, 7),
ReplicationTaskSnapshotSuccess("src", "data/src/src2",
"2018-10-01_03-00", 6, 7),
ReplicationTaskSnapshotStart("src", "data/src/src2",
"2018-10-01_04-00", 6, 7),
ReplicationTaskSnapshotSuccess("src", "data/src/src2",
"2018-10-01_04-00", 7, 7),
ReplicationTaskSuccess("src"),
]
if transport["type"] == "ssh":
result.insert(
4,
ReplicationTaskSnapshotProgress(
"src",
"data/src/src1",
"2018-10-01_02-00",
1,
3,
10240 * 9 * 10, # We poll for progress every 10 seconds so
# we would have transferred 10x speed limit
2162784 # Empirical value
))
for i, message in enumerate(result):
call = calls[i]
assert call[0][0].__class__ == message.__class__, calls
d1 = call[0][0].__dict__
d2 = message.__dict__
if isinstance(message, ReplicationTaskSnapshotProgress):
bytes_sent_1 = d1.pop("bytes_sent")
bytes_total_1 = d1.pop("bytes_total")
bytes_sent_2 = d2.pop("bytes_sent")
bytes_total_2 = d2.pop("bytes_total")
assert 0.8 <= bytes_sent_1 / bytes_sent_2 <= 1.2
assert 0.8 <= bytes_total_1 / bytes_total_2 <= 1.2
assert d1 == d2
def test_replication_retry(caplog):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
transport:
type: ssh
hostname: localhost
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: false
retention-policy: none
retries: 2
"""))
set_localhost_transport_options(
definition["replication-tasks"]["src"]["transport"])
definition["replication-tasks"]["src"]["transport"][
"private-key"] = textwrap.dedent("""\
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA0/5hQu83T9Jdl1NT9malC0ovHMHLspa4t6dFTSHWRUHsA3+t
q50bBfrsS+hm4qMndxm9Sqig5/TqlM00W49SkooyU/0j4Q4xjvJ61RXOtHXPOoMH
opLjRlmbuxkWCb0CmwXvIunaebBFfPx/VuwNJNNv9ZNcgeQJj5ggjI7hnikK4Pn4
jpqcivqIStNO/6q+9NLsNkMQu8vq/zuxC9ePyeaywbbAIcpKREsWgiNtuhsPxnRS
+gVQ+XVgE6RFJzMO13MtE+E4Uphseip+fSNVmLeAQyGUrUg12JevJYnMbLOQtacB
GNDMHSwcwAzqVYPq8oqjQhWvqBntjcd/qK3P+wIDAQABAoIBAHy8tzoNS7x6CXvb
GhJn/0EPW31OQq9IpFPb5pkmCdAio97DJ8tM2/O+238mtjMw0S3xRUJCyrrxj34S
6HXfdTSogEiPMKdiFKMJ5mCvPjtM/qxtIPb1+ykP3ORQNHlyb7AL49PlShpEL/8F
C2B38Jv0lXIoTUxYg4+scaqDABpw9aaYTODcJ9uvFhAcAHALKaN0iiz050dWoH9D
CkJ1UwoHVUz6XGZ3lOR/qxUDGd72Ara0cizCXQZIkOtu8Kfnfnlx3pqOZJgbkr49
JY3LQId5bVhNlQLKlTSAameIiAJETeLvxHzJHCvMm0LnKDfLiejq/dEk5CMgjrVz
ExV+ioECgYEA72zxquQJo051o2mrG0DhVBT0QzXo+8yjNYVha2stBOMGvEnL0n2H
VFDdWhpZVzRs1uR6sJC14YTGfBNk7NTaQSorgrKvYs1E/krZEMsFquwIcLtbHxYP
zjBSQwYA7jIEFViIkZwptb+qfA+c1YehZTYzx4R/hlkkLlTObyRFcyECgYEA4qtK
/7UaBG4kumW+cdRnqJ+KO21PylBnGaCm6yH6DO5SKlqoHvYdyds70Oat9fPX4BRJ
2aMTivZMkGgu6Dc1AViRgBoTIReMQ9TY3y8d0unMtBddAIx0guiP/rtPrCRTC07m
s31b6wkLTnPnW3W2N8t4LfdTLpsgmA3t5Q6Iu5sCgYB9Lg+4kpu7Z3U4KDJPAIAP
Lxl63n/ezuJyRDdoK1QRXwWRgl/vwLP10IW661XUs1NIk5LWKAMAUyRXkOhOrwch
1QOExRnP5ZTyA340OoHPGLNdBYgh264N1tPbuRLZdwsNggl9YBGqtfhT/vG37r7i
pREzesIWIxs4ohyAnY02IQKBgQDARd0Qm2a+a0/sbXHmzO5BM1PmpQsR6rIKIyR0
QBYD8gTwuIXz/YG3QKi0w3i9MWLlSVB7tMFXFyZLOJTRlkL4KVEDARtI7tikkWCF
sUnzJy/ldAwH8xzCDtRWmD01IHrxFLTNfIEEFl/o5JhUFL3FBmujUjDVT/GOCgLK
UlHaEQKBgFUGEgI6/GvV/JecnEWLqd+HpRHiBywpfOkAFmJGokdAOvF0QDFHK9/P
stO7TRqUHufxZQIeTJ7sGdsabEAypiKSFBR8w1qVg+iQZ+M+t0vCgXlnHLaw2SeJ
1YT8kH1TsdzozkxJ7tFa1A5YI37ZiUiN7ykJ0l4Zal6Nli9z5Oa0
-----END RSA PRIVATE KEY-----
""") # Some random invalid SSH key
definition = Definition.from_data(definition)
caplog.set_level(logging.INFO)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert any("non-recoverable replication error" in record.message
for record in caplog.get_records("call"))
def test_replication_progress_pre_calculate():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/alice", shell=True)
subprocess.check_call("zfs create data/src/bob", shell=True)
subprocess.check_call("zfs create data/src/charlie", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call(
"zfs send -R data/src@2018-10-01_01-00 | zfs recv -s -F data/dst",
shell=True)
subprocess.check_call("zfs create data/src/dave", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
calls = [
call for call in zettarepl.observer.call_args_list
if call[0][0].__class__ != ReplicationTaskDataProgress
]
result = [
ReplicationTaskStart("src"),
ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_02-00", 0,
5),
ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_02-00",
1, 5),
ReplicationTaskSnapshotStart("src", "data/src/alice",
"2018-10-01_02-00", 1, 5),
ReplicationTaskSnapshotSuccess("src", "data/src/alice",
"2018-10-01_02-00", 2, 5),
ReplicationTaskSnapshotStart("src", "data/src/bob", "2018-10-01_02-00",
2, 5),
ReplicationTaskSnapshotSuccess("src", "data/src/bob",
"2018-10-01_02-00", 3, 5),
ReplicationTaskSnapshotStart("src", "data/src/charlie",
"2018-10-01_02-00", 3, 5),
ReplicationTaskSnapshotSuccess("src", "data/src/charlie",
"2018-10-01_02-00", 4, 5),
ReplicationTaskSnapshotStart("src", "data/src/dave",
"2018-10-01_02-00", 4, 5),
ReplicationTaskSnapshotSuccess("src", "data/src/dave",
"2018-10-01_02-00", 5, 5),
ReplicationTaskSuccess("src"),
]
for i, message in enumerate(result):
call = calls[i]
assert call[0][0].__class__ == message.__class__
d1 = call[0][0].__dict__
d2 = message.__dict__
assert d1 == d2
def test_replication_retry(caplog, direction):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
transport:
type: ssh
hostname: 127.0.0.1
source-dataset: data/src
target-dataset: data/dst
recursive: true
auto: false
retention-policy: none
speed-limit: 200000
retries: 2
"""))
definition["replication-tasks"]["src"]["direction"] = direction
if direction == "push":
definition["replication-tasks"]["src"]["periodic-snapshot-tasks"] = [
"src"
]
else:
definition["replication-tasks"]["src"]["naming-schema"] = [
"%Y-%m-%d_%H-%M"
]
set_localhost_transport_options(
definition["replication-tasks"]["src"]["transport"])
definition = Definition.from_data(definition)
caplog.set_level(logging.INFO)
zettarepl = create_zettarepl(definition)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
time.sleep(2)
if direction == "push":
subprocess.check_output("kill $(pgrep -f '^zfs recv')", shell=True)
else:
subprocess.check_output("kill $(pgrep -f '^(zfs send|zfs: sending)')",
shell=True)
wait_replication_tasks_to_complete(zettarepl)
assert any(" recoverable replication error" in record.message
for record in caplog.get_records("call"))
assert any("Resuming replication for destination dataset" in record.message
for record in caplog.get_records("call"))
success = zettarepl.observer.call_args_list[-1][0][0]
assert isinstance(success, ReplicationTaskSuccess), success
local_shell = LocalShell()
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
