async def get_definition(self): config = await self.middleware.call("replication.config.config") timezone = (await self.middleware.call("system.general.config"))["timezone"] pools = {pool["name"]: pool for pool in await self.middleware.call("pool.query")} hold_tasks = {} periodic_snapshot_tasks = {} for periodic_snapshot_task in await self.middleware.call("pool.snapshottask.query", [["enabled", "=", True]]): hold_task_reason = self._hold_task_reason(pools, periodic_snapshot_task["dataset"]) if hold_task_reason: hold_tasks[f"periodic_snapshot_task_{periodic_snapshot_task['id']}"] = hold_task_reason continue periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = self.periodic_snapshot_task_definition( periodic_snapshot_task, ) replication_tasks = {} for replication_task in await self.middleware.call("replication.query", [["enabled", "=", True]]): try: replication_tasks[f"task_{replication_task['id']}"] = await self._replication_task_definition( pools, replication_task ) except HoldReplicationTaskException as e: hold_tasks[f"replication_task_{replication_task['id']}"] = e.reason for job_id, replication_task in self.onetime_replication_tasks.items(): try: replication_tasks[f"job_{job_id}"] = await self._replication_task_definition(pools, replication_task) except HoldReplicationTaskException as e: hold_tasks[f"job_{job_id}"] = e.reason definition = { "max-parallel-replication-tasks": config["max_parallel_replication_tasks"], "timezone": timezone, "use-removal-dates": True, "periodic-snapshot-tasks": periodic_snapshot_tasks, "replication-tasks": replication_tasks, } # Test if does not cause exceptions Definition.from_data(definition, raise_on_error=False) hold_tasks = { task_id: { "state": "HOLD", "datetime": datetime.utcnow(), "reason": make_sentence(reason), } for task_id, reason in hold_tasks.items() } return definition, hold_tasks
def test_push_replication(transport, properties): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["properties"] = properties definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 assert ( ("test-value" in subprocess.check_output("zfs get test:property data/dst", shell=True, encoding="utf-8")) == properties ) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "config": if "max_parallel_replication_tasks" in args: self.zettarepl.max_parallel_replication_tasks = args[ "max_parallel_replication_tasks"] if "timezone" in args: self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args["timezone"]) if command == "tasks": definition = Definition.from_data(args, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) self.zettarepl.set_tasks(definition.tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id) self.observer_queue.put( ReplicationTaskError(task_id, "Task not found"))
def __call__(self): if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl debug_level = "DEBUG" elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl debug_level = "INFO" else: debug_level = self.debug_level setup_logging("zettarepl", debug_level, self.log_handler) definition = Definition.from_data(self.definition) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10)
def test_does_not_remove_the_last_snapshot_left(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2020-05-07_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2020-05-23_00-00", shell=True) data = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: false naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" lifetime: P30D """)) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2020, 6, 25, 0, 0)) assert list_snapshots(local_shell, "data/src", False) == [Snapshot("data/src", "2020-05-23_00-00")]
def test_push_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/child", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """)) definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 1 subprocess.check_call("zfs destroy -r data/src/child", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 2 local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/dst/child", False)) == 1
def test_replication_resume(caplog): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call( "dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call( "(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output( "zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: - id: src dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: - id: src direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) assert any("Resuming replication for dst_dataset" in record.message for record in caplog.get_records("call")) assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def run_periodic_snapshot_test(definition, now, success=True): definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._run_periodic_snapshot_tasks(now, select_by_class(PeriodicSnapshotTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) if success: for call in zettarepl.observer.call_args_list: call = call[0][0] assert not isinstance(call, PeriodicSnapshotTaskError), success
def test_replication_data_progress(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: ssh hostname: 127.0.0.1 source-dataset: - data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) set_localhost_transport_options( definition["replication-tasks"]["src"]["transport"]) definition["replication-tasks"]["src"]["speed-limit"] = 10240 * 9 with patch("zettarepl.replication.run.DatasetSizeObserver.INTERVAL", 5): definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) calls = [ call for call in zettarepl.observer.call_args_list if call[0][0].__class__ == ReplicationTaskDataProgress ] assert len(calls) == 2 assert 1024 * 1024 * 0.8 <= calls[0][0][0].src_size <= 1024 * 1024 * 1.2 assert 0 <= calls[0][0][0].dst_size <= 10240 * 1.2 assert 1024 * 1024 * 0.8 <= calls[1][0][0].src_size <= 1024 * 1024 * 1.2 assert 10240 * 6 * 0.8 <= calls[1][0][0].dst_size <= 10240 * 6 * 1.2
def test_push_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = Definition.from_data(yaml.load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def run_replication_test(definition, success=True): definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) if success: success = zettarepl.observer.call_args_list[-1][0][0] assert isinstance(success, ReplicationTaskSuccess), success else: error = zettarepl.observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskError), error return error
def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') osc.die_with_parent() if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl default_level = logging.DEBUG elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl default_level = logging.INFO else: default_level = logging.getLevelName(self.debug_level) setup_logging("", "DEBUG", self.log_handler) oqlh = ObserverQueueLoggingHandler(self.observer_queue) oqlh.setFormatter( logging.Formatter( '[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s', '%Y/%m/%d %H:%M:%S')) logging.getLogger("zettarepl").addHandler(oqlh) for handler in logging.getLogger("zettarepl").handlers: handler.addFilter(LongStringsFilter()) handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level)) c = Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) c.subscribe( 'core.reconfigure_logging', lambda *args, **kwargs: reconfigure_logging('zettarepl_file')) definition = Definition.from_data(self.definition, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10)
def test_source_retention_multiple_sources(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_02-00", shell=True) definition = Definition.from_data(yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: [data/src/a, data/src/b] target-dataset: data/dst recursive: false periodic-snapshot-tasks: - src auto: true retention-policy: source hold-pending-snapshots: true """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst/a", False) == [Snapshot("data/dst/a", "2018-10-01_02-00")] assert list_snapshots(local_shell, "data/dst/b", False) == [Snapshot("data/dst/b", "2018-10-01_02-00")]
def test_push_remote_retention(retention_policy, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True) data = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true """)) data["replication-tasks"]["src"].update(**retention_policy) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst", False) == remains
def test_hold_pending_snapshots(hold_pending_snapshots, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: - id: src dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: - id: src direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: source hold-pending-snapshots: """ + yaml.dump(hold_pending_snapshots) + """ """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/src", False) == remains
def test_zvol_replication(as_root): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) if as_root: subprocess.check_call("zfs create -V 1M data/src", shell=True) else: subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create -V 1M data/src/zvol", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 if not as_root: assert len(list_snapshots(local_shell, "data/dst/zvol", False)) == 2
def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(args) if command == "tasks": self.zettarepl.set_tasks(Definition.from_data(args).tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id)
def test_hold_pending_snapshots(retention_policy, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True) data = yaml.load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: - id: src direction: pull transport: type: local source-dataset: data/src target-dataset: data/dst naming-schema: "%Y-%m-%d_%H-%M" recursive: true auto: true """)) data["replication-tasks"][0].update(**retention_policy) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst", False) == remains
def test_does_not_remove_the_last_snapshot_left(snapshots__removal_dates__result): snapshots, removal_dates, result = snapshots__removal_dates__result subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/src2", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/child", shell=True) subprocess.check_call("zfs create data/src2", shell=True) for snapshot in snapshots: subprocess.check_call(f"zfs snapshot {snapshot}", shell=True) data = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: false naming-schema: "%Y-%m-%d-%H-%M" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" lifetime: P30D """)) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell, use_removal_dates=True) zettarepl.set_tasks(definition.tasks) with patch("zettarepl.zettarepl.get_removal_dates", Mock(return_value=removal_dates)): zettarepl._run_local_retention(datetime(2021, 4, 19, 17, 0)) assert list_snapshots(local_shell, "data/src", False) + list_snapshots(local_shell, "data/src2", False) == [ snapshots[i] for i in result ]
def run(args): try: definition = Definition.from_data(yaml.load(args.definition_path)) except yaml.YAMLError as e: sys.stderr.write(f"Definition syntax error: {e!s}\n") sys.exit(1) except jsonschema.exceptions.ValidationError as e: sys.stderr.write(f"Definition validation error: {e!s}\n") sys.exit(1) except ValueError as e: sys.stderr.write(f"{e!s}\n") sys.exit(1) clock = Clock(args.once) tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() zettarepl = Zettarepl(scheduler, local_shell) zettarepl.set_tasks(definition.tasks) zettarepl.run()
def test_pull_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: - id: src direction: pull transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true naming-schema: - "%Y-%m-%d_%H-%M" auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) assert len(list_snapshots(local_shell, "data/dst", False)) == 2
def test_replication_progress_resume(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_04-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call( "zfs send data/src@2018-10-01_01-00 | zfs recv -s -F data/dst", shell=True) subprocess.check_call( "(zfs send -i data/src@2018-10-01_01-00 data/src@2018-10-01_02-00 | " " throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output( "zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) calls = [ call for call in zettarepl.observer.call_args_list if call[0][0].__class__ != ReplicationTaskDataProgress ] result = [ ReplicationTaskStart("src"), ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_02-00", 0, 3), ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_02-00", 1, 3), ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_03-00", 1, 3), ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_03-00", 2, 3), ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_04-00", 2, 3), ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_04-00", 3, 3), ReplicationTaskSuccess("src"), ] for i, message in enumerate(result): call = calls[i] assert call[0][0].__class__ == message.__class__ d1 = call[0][0].__dict__ d2 = message.__dict__ assert d1 == d2
def test_parallel_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src-a: dataset: data/src/a recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" src-b: dataset: data/src/b recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src-a: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/a target-dataset: data/dst/a recursive: true periodic-snapshot-tasks: - src-a auto: true retention-policy: none speed-limit: 100000 src-b: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/b target-dataset: data/dst/b recursive: true periodic-snapshot-tasks: - src-b auto: true retention-policy: none speed-limit: 100000 """)) set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"]) set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"]) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 10 <= end - start <= 15 zettarepl._spawn_retention.assert_called_once() assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 2 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1 subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 20 <= end - start <= 25 assert sum(1 for m in zettarepl.observer.call_args_list if isinstance(m[0][0], ReplicationTaskSuccess)) == 4 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
async def get_definition(self): timezone = (await self.middleware.call("system.general.config"))["timezone"] pools = { pool["name"]: pool for pool in await self.middleware.call("pool.query") } hold_tasks = {} periodic_snapshot_tasks = {} for periodic_snapshot_task in await self.middleware.call( "pool.snapshottask.query", [["enabled", "=", True]]): hold_task_reason = self._hold_task_reason( pools, periodic_snapshot_task["dataset"]) if hold_task_reason: hold_tasks[ f"periodic_snapshot_task_{periodic_snapshot_task['id']}"] = hold_task_reason continue periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = { "dataset": periodic_snapshot_task["dataset"], "recursive": periodic_snapshot_task["recursive"], "exclude": periodic_snapshot_task["exclude"], "lifetime": lifetime_iso8601(periodic_snapshot_task["lifetime_value"], periodic_snapshot_task["lifetime_unit"]), "naming-schema": periodic_snapshot_task["naming_schema"], "schedule": zettarepl_schedule(periodic_snapshot_task["schedule"]), "allow-empty": periodic_snapshot_task["allow_empty"], } replication_tasks = {} for replication_task in await self.middleware.call( "replication.query", [["enabled", "=", True]]): if replication_task["direction"] == "PUSH": hold = False for source_dataset in replication_task["source_datasets"]: hold_task_reason = self._hold_task_reason( pools, source_dataset) if hold_task_reason: hold_tasks[ f"replication_task_{replication_task['id']}"] = hold_task_reason hold = True break if hold: continue if replication_task["direction"] == "PULL": hold_task_reason = self._hold_task_reason( pools, replication_task["target_dataset"]) if hold_task_reason: hold_tasks[ f"replication_task_{replication_task['id']}"] = hold_task_reason continue if replication_task["transport"] != "LOCAL": if not await self.middleware.call( "network.general.can_perform_activity", "replication"): hold_tasks[ f"replication_task_{replication_task['id']}"] = ( "Replication network activity is disabled") continue try: transport = await self._define_transport( replication_task["transport"], (replication_task["ssh_credentials"] or {}).get("id"), replication_task["netcat_active_side"], replication_task["netcat_active_side_listen_address"], replication_task["netcat_active_side_port_min"], replication_task["netcat_active_side_port_max"], replication_task["netcat_passive_side_connect_address"], ) except CallError as e: hold_tasks[ f"replication_task_{replication_task['id']}"] = e.errmsg continue my_periodic_snapshot_tasks = [ f"task_{periodic_snapshot_task['id']}" for periodic_snapshot_task in replication_task["periodic_snapshot_tasks"] ] my_schedule = replication_task["schedule"] definition = { "direction": replication_task["direction"].lower(), "transport": transport, "source-dataset": replication_task["source_datasets"], "target-dataset": replication_task["target_dataset"], "recursive": replication_task["recursive"], "exclude": replication_task_exclude(replication_task), "properties": replication_task["properties"], "properties-exclude": replication_task["properties_exclude"], "properties-override": replication_task["properties_override"], "replicate": replication_task["replicate"], "periodic-snapshot-tasks": my_periodic_snapshot_tasks, "auto": replication_task["auto"], "only-matching-schedule": replication_task["only_matching_schedule"], "allow-from-scratch": replication_task["allow_from_scratch"], "readonly": replication_task["readonly"].lower(), "hold-pending-snapshots": replication_task["hold_pending_snapshots"], "retention-policy": replication_task["retention_policy"].lower(), "large-block": replication_task["large_block"], "embed": replication_task["embed"], "compressed": replication_task["compressed"], "retries": replication_task["retries"], "logging-level": (replication_task["logging_level"] or "NOTSET").lower(), } if replication_task["encryption"]: definition["encryption"] = { "key": replication_task["encryption_key"], "key-format": replication_task["encryption_key_format"], "key-location": replication_task["encryption_key_location"], } if replication_task["naming_schema"]: definition["naming-schema"] = replication_task["naming_schema"] if replication_task["also_include_naming_schema"]: definition["also-include-naming-schema"] = replication_task[ "also_include_naming_schema"] if my_schedule is not None: definition["schedule"] = zettarepl_schedule(my_schedule) if replication_task["restrict_schedule"] is not None: definition["restrict-schedule"] = zettarepl_schedule( replication_task["restrict_schedule"]) if replication_task[ "lifetime_value"] is not None and replication_task[ "lifetime_unit"] is not None: definition["lifetime"] = lifetime_iso8601( replication_task["lifetime_value"], replication_task["lifetime_unit"]) if replication_task["compression"] is not None: definition["compression"] = replication_task[ "compression"].lower() if replication_task["speed_limit"] is not None: definition["speed-limit"] = replication_task["speed_limit"] replication_tasks[f"task_{replication_task['id']}"] = definition definition = { "timezone": timezone, "periodic-snapshot-tasks": periodic_snapshot_tasks, "replication-tasks": replication_tasks, } # Test if does not cause exceptions Definition.from_data(definition, raise_on_error=False) hold_tasks = { task_id: { "state": "HOLD", "datetime": datetime.utcnow(), "reason": make_sentence(reason), } for task_id, reason in hold_tasks.items() } return definition, hold_tasks
async def get_definition(self): timezone = (await self.middleware.call("system.general.config"))["timezone"] periodic_snapshot_tasks = {} for periodic_snapshot_task in await self.middleware.call( "pool.snapshottask.query", [["enabled", "=", True], ["legacy", "=", False]]): periodic_snapshot_tasks[f"task_{periodic_snapshot_task['id']}"] = { "dataset": periodic_snapshot_task["dataset"], "recursive": periodic_snapshot_task["recursive"], "exclude": periodic_snapshot_task["exclude"], "lifetime": lifetime_iso8601(periodic_snapshot_task["lifetime_value"], periodic_snapshot_task["lifetime_unit"]), "naming-schema": periodic_snapshot_task["naming_schema"], "schedule": zettarepl_schedule(periodic_snapshot_task["schedule"]), "allow-empty": periodic_snapshot_task["allow_empty"], } replication_tasks = {} legacy_periodic_snapshot_tasks_ids = { periodic_snapshot_task["id"] for periodic_snapshot_task in await self.middleware.call( "pool.snapshottask.query", [["legacy", "=", True]]) } for replication_task in await self.middleware.call( "replication.query", [["transport", "!=", "LEGACY"], ["enabled", "=", True]]): my_periodic_snapshot_tasks = [ f"task_{periodic_snapshot_task['id']}" for periodic_snapshot_task in replication_task["periodic_snapshot_tasks"] if periodic_snapshot_task["id"] not in legacy_periodic_snapshot_tasks_ids ] my_schedule = replication_task["schedule"] # All my periodic snapshot tasks are legacy if (replication_task["direction"] == "PUSH" and replication_task["auto"] and replication_task["periodic_snapshot_tasks"] and not my_periodic_snapshot_tasks): my_schedule = replication_task["periodic_snapshot_tasks"][0][ "schedule"] definition = { "direction": replication_task["direction"].lower(), "transport": await self._define_transport( replication_task["transport"], (replication_task["ssh_credentials"] or {}).get("id"), replication_task["netcat_active_side"], replication_task["netcat_active_side_listen_address"], replication_task["netcat_active_side_port_min"], replication_task["netcat_active_side_port_max"], replication_task["netcat_passive_side_connect_address"], ), "source-dataset": replication_task["source_datasets"], "target-dataset": replication_task["target_dataset"], "recursive": replication_task["recursive"], "exclude": replication_task_exclude(replication_task["source_datasets"], replication_task["recursive"], replication_task["exclude"]), "properties": replication_task["properties"], "periodic-snapshot-tasks": my_periodic_snapshot_tasks, "auto": replication_task["auto"], "only-matching-schedule": replication_task["only_matching_schedule"], "allow-from-scratch": replication_task["allow_from_scratch"], "hold-pending-snapshots": replication_task["hold_pending_snapshots"], "retention-policy": replication_task["retention_policy"].lower(), "dedup": replication_task["dedup"], "large-block": replication_task["large_block"], "embed": replication_task["embed"], "compressed": replication_task["compressed"], "retries": replication_task["retries"], "logging-level": (replication_task["logging_level"] or "NOTSET").lower(), } if replication_task["naming_schema"]: definition["naming-schema"] = replication_task["naming_schema"] if replication_task["also_include_naming_schema"]: definition["also-include-naming-schema"] = replication_task[ "also_include_naming_schema"] # Use snapshots created by legacy periodic snapshot tasks for periodic_snapshot_task in replication_task[ "periodic_snapshot_tasks"]: if periodic_snapshot_task[ "id"] in legacy_periodic_snapshot_tasks_ids: definition.setdefault("also-include-naming-schema", []) definition["also-include-naming-schema"].append( periodic_snapshot_task["naming_schema"]) if my_schedule is not None: definition["schedule"] = zettarepl_schedule(my_schedule) if replication_task["restrict_schedule"] is not None: definition["restrict-schedule"] = zettarepl_schedule( replication_task["restrict_schedule"]) if replication_task[ "lifetime_value"] is not None and replication_task[ "lifetime_unit"] is not None: definition["lifetime"] = lifetime_iso8601( replication_task["lifetime_value"], replication_task["lifetime_unit"]) if replication_task["compression"] is not None: definition["compression"] = replication_task["compression"] if replication_task["speed_limit"] is not None: definition["speed-limit"] = replication_task["speed_limit"] replication_tasks[f"task_{replication_task['id']}"] = definition definition = { "timezone": timezone, "periodic-snapshot-tasks": periodic_snapshot_tasks, "replication-tasks": replication_tasks, } # Test if does not cause exceptions Definition.from_data(definition, raise_on_error=False) return definition
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, properties, compression): if transport["type"] != "ssh" and compression: return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst_parent", shell=True) if dst_exists: subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if dst_parent_is_readonly: subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["properties"] = properties if compression: definition["replication-tasks"]["src"]["compression"] = compression definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2 assert (("test-value" in subprocess.check_output( "zfs get test:property data/dst_parent/dst", shell=True, encoding="utf-8")) == properties) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_replication_progress(transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/src1", shell=True) subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_01-00", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/src1/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_02-00", shell=True) subprocess.check_call("rm /mnt/data/src/src1/blob", shell=True) subprocess.check_call("zfs snapshot data/src/src1@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/src/src2", shell=True) subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_03-00", shell=True) subprocess.check_call("zfs snapshot data/src/src2@2018-10-01_04-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push source-dataset: - data/src/src1 - data/src/src2 target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport if transport["type"] == "ssh": definition["replication-tasks"]["src"]["speed-limit"] = 10240 * 9 definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) calls = [ call for call in zettarepl.observer.call_args_list if call[0][0].__class__ != ReplicationTaskDataProgress ] result = [ ReplicationTaskStart("src"), ReplicationTaskSnapshotStart("src", "data/src/src1", "2018-10-01_01-00", 0, 3), ReplicationTaskSnapshotSuccess("src", "data/src/src1", "2018-10-01_01-00", 1, 3), ReplicationTaskSnapshotStart("src", "data/src/src1", "2018-10-01_02-00", 1, 3), ReplicationTaskSnapshotSuccess("src", "data/src/src1", "2018-10-01_02-00", 2, 3), ReplicationTaskSnapshotStart("src", "data/src/src1", "2018-10-01_03-00", 2, 3), ReplicationTaskSnapshotSuccess("src", "data/src/src1", "2018-10-01_03-00", 3, 3), ReplicationTaskSnapshotStart("src", "data/src/src2", "2018-10-01_01-00", 3, 7), ReplicationTaskSnapshotSuccess("src", "data/src/src2", "2018-10-01_01-00", 4, 7), ReplicationTaskSnapshotStart("src", "data/src/src2", "2018-10-01_02-00", 4, 7), ReplicationTaskSnapshotSuccess("src", "data/src/src2", "2018-10-01_02-00", 5, 7), ReplicationTaskSnapshotStart("src", "data/src/src2", "2018-10-01_03-00", 5, 7), ReplicationTaskSnapshotSuccess("src", "data/src/src2", "2018-10-01_03-00", 6, 7), ReplicationTaskSnapshotStart("src", "data/src/src2", "2018-10-01_04-00", 6, 7), ReplicationTaskSnapshotSuccess("src", "data/src/src2", "2018-10-01_04-00", 7, 7), ReplicationTaskSuccess("src"), ] if transport["type"] == "ssh": result.insert( 4, ReplicationTaskSnapshotProgress( "src", "data/src/src1", "2018-10-01_02-00", 1, 3, 10240 * 9 * 10, # We poll for progress every 10 seconds so # we would have transferred 10x speed limit 2162784 # Empirical value )) for i, message in enumerate(result): call = calls[i] assert call[0][0].__class__ == message.__class__, calls d1 = call[0][0].__dict__ d2 = message.__dict__ if isinstance(message, ReplicationTaskSnapshotProgress): bytes_sent_1 = d1.pop("bytes_sent") bytes_total_1 = d1.pop("bytes_total") bytes_sent_2 = d2.pop("bytes_sent") bytes_total_2 = d2.pop("bytes_total") assert 0.8 <= bytes_sent_1 / bytes_sent_2 <= 1.2 assert 0.8 <= bytes_total_1 / bytes_total_2 <= 1.2 assert d1 == d2
def test_replication_retry(caplog): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: transport: type: ssh hostname: localhost direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: false retention-policy: none retries: 2 """)) set_localhost_transport_options( definition["replication-tasks"]["src"]["transport"]) definition["replication-tasks"]["src"]["transport"][ "private-key"] = textwrap.dedent("""\ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA0/5hQu83T9Jdl1NT9malC0ovHMHLspa4t6dFTSHWRUHsA3+t q50bBfrsS+hm4qMndxm9Sqig5/TqlM00W49SkooyU/0j4Q4xjvJ61RXOtHXPOoMH opLjRlmbuxkWCb0CmwXvIunaebBFfPx/VuwNJNNv9ZNcgeQJj5ggjI7hnikK4Pn4 jpqcivqIStNO/6q+9NLsNkMQu8vq/zuxC9ePyeaywbbAIcpKREsWgiNtuhsPxnRS +gVQ+XVgE6RFJzMO13MtE+E4Uphseip+fSNVmLeAQyGUrUg12JevJYnMbLOQtacB GNDMHSwcwAzqVYPq8oqjQhWvqBntjcd/qK3P+wIDAQABAoIBAHy8tzoNS7x6CXvb GhJn/0EPW31OQq9IpFPb5pkmCdAio97DJ8tM2/O+238mtjMw0S3xRUJCyrrxj34S 6HXfdTSogEiPMKdiFKMJ5mCvPjtM/qxtIPb1+ykP3ORQNHlyb7AL49PlShpEL/8F C2B38Jv0lXIoTUxYg4+scaqDABpw9aaYTODcJ9uvFhAcAHALKaN0iiz050dWoH9D CkJ1UwoHVUz6XGZ3lOR/qxUDGd72Ara0cizCXQZIkOtu8Kfnfnlx3pqOZJgbkr49 JY3LQId5bVhNlQLKlTSAameIiAJETeLvxHzJHCvMm0LnKDfLiejq/dEk5CMgjrVz ExV+ioECgYEA72zxquQJo051o2mrG0DhVBT0QzXo+8yjNYVha2stBOMGvEnL0n2H VFDdWhpZVzRs1uR6sJC14YTGfBNk7NTaQSorgrKvYs1E/krZEMsFquwIcLtbHxYP zjBSQwYA7jIEFViIkZwptb+qfA+c1YehZTYzx4R/hlkkLlTObyRFcyECgYEA4qtK /7UaBG4kumW+cdRnqJ+KO21PylBnGaCm6yH6DO5SKlqoHvYdyds70Oat9fPX4BRJ 2aMTivZMkGgu6Dc1AViRgBoTIReMQ9TY3y8d0unMtBddAIx0guiP/rtPrCRTC07m s31b6wkLTnPnW3W2N8t4LfdTLpsgmA3t5Q6Iu5sCgYB9Lg+4kpu7Z3U4KDJPAIAP Lxl63n/ezuJyRDdoK1QRXwWRgl/vwLP10IW661XUs1NIk5LWKAMAUyRXkOhOrwch 1QOExRnP5ZTyA340OoHPGLNdBYgh264N1tPbuRLZdwsNggl9YBGqtfhT/vG37r7i pREzesIWIxs4ohyAnY02IQKBgQDARd0Qm2a+a0/sbXHmzO5BM1PmpQsR6rIKIyR0 QBYD8gTwuIXz/YG3QKi0w3i9MWLlSVB7tMFXFyZLOJTRlkL4KVEDARtI7tikkWCF sUnzJy/ldAwH8xzCDtRWmD01IHrxFLTNfIEEFl/o5JhUFL3FBmujUjDVT/GOCgLK UlHaEQKBgFUGEgI6/GvV/JecnEWLqd+HpRHiBywpfOkAFmJGokdAOvF0QDFHK9/P stO7TRqUHufxZQIeTJ7sGdsabEAypiKSFBR8w1qVg+iQZ+M+t0vCgXlnHLaw2SeJ 1YT8kH1TsdzozkxJ7tFa1A5YI37ZiUiN7ykJ0l4Zal6Nli9z5Oa0 -----END RSA PRIVATE KEY----- """) # Some random invalid SSH key definition = Definition.from_data(definition) caplog.set_level(logging.INFO) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert any("non-recoverable replication error" in record.message for record in caplog.get_records("call"))
def test_replication_progress_pre_calculate(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/alice", shell=True) subprocess.check_call("zfs create data/src/bob", shell=True) subprocess.check_call("zfs create data/src/charlie", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call( "zfs send -R data/src@2018-10-01_01-00 | zfs recv -s -F data/dst", shell=True) subprocess.check_call("zfs create data/src/dave", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) calls = [ call for call in zettarepl.observer.call_args_list if call[0][0].__class__ != ReplicationTaskDataProgress ] result = [ ReplicationTaskStart("src"), ReplicationTaskSnapshotStart("src", "data/src", "2018-10-01_02-00", 0, 5), ReplicationTaskSnapshotSuccess("src", "data/src", "2018-10-01_02-00", 1, 5), ReplicationTaskSnapshotStart("src", "data/src/alice", "2018-10-01_02-00", 1, 5), ReplicationTaskSnapshotSuccess("src", "data/src/alice", "2018-10-01_02-00", 2, 5), ReplicationTaskSnapshotStart("src", "data/src/bob", "2018-10-01_02-00", 2, 5), ReplicationTaskSnapshotSuccess("src", "data/src/bob", "2018-10-01_02-00", 3, 5), ReplicationTaskSnapshotStart("src", "data/src/charlie", "2018-10-01_02-00", 3, 5), ReplicationTaskSnapshotSuccess("src", "data/src/charlie", "2018-10-01_02-00", 4, 5), ReplicationTaskSnapshotStart("src", "data/src/dave", "2018-10-01_02-00", 4, 5), ReplicationTaskSnapshotSuccess("src", "data/src/dave", "2018-10-01_02-00", 5, 5), ReplicationTaskSuccess("src"), ] for i, message in enumerate(result): call = calls[i] assert call[0][0].__class__ == message.__class__ d1 = call[0][0].__dict__ d2 = message.__dict__ assert d1 == d2
def test_replication_retry(caplog, direction): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: transport: type: ssh hostname: 127.0.0.1 source-dataset: data/src target-dataset: data/dst recursive: true auto: false retention-policy: none speed-limit: 200000 retries: 2 """)) definition["replication-tasks"]["src"]["direction"] = direction if direction == "push": definition["replication-tasks"]["src"]["periodic-snapshot-tasks"] = [ "src" ] else: definition["replication-tasks"]["src"]["naming-schema"] = [ "%Y-%m-%d_%H-%M" ] set_localhost_transport_options( definition["replication-tasks"]["src"]["transport"]) definition = Definition.from_data(definition) caplog.set_level(logging.INFO) zettarepl = create_zettarepl(definition) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) time.sleep(2) if direction == "push": subprocess.check_output("kill $(pgrep -f '^zfs recv')", shell=True) else: subprocess.check_output("kill $(pgrep -f '^(zfs send|zfs: sending)')", shell=True) wait_replication_tasks_to_complete(zettarepl) assert any(" recoverable replication error" in record.message for record in caplog.get_records("call")) assert any("Resuming replication for destination dataset" in record.message for record in caplog.get_records("call")) success = zettarepl.observer.call_args_list[-1][0][0] assert isinstance(success, ReplicationTaskSuccess), success local_shell = LocalShell() assert len(list_snapshots(local_shell, "data/dst", False)) == 1