def test__can_run_in_parallel__different_direction(t1, t2, can): t1.direction = ReplicationDirection.PUSH t2.direction = ReplicationDirection.PULL zettarepl = Zettarepl(Mock(), Mock()) assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def __call__(self): if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl debug_level = "DEBUG" elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl debug_level = "INFO" else: debug_level = self.debug_level setup_logging("zettarepl", debug_level, self.log_handler) definition = Definition.from_data(self.definition) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10)
def test__can_run_in_parallel__same_direction(t1, t2, can): with patch("zettarepl.zettarepl.are_same_host", Mock(return_value=True)): t1.direction = t2.direction = None zettarepl = Zettarepl(Mock(), Mock()) assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def test_push_replication(transport, properties): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["properties"] = properties definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 assert ( ("test-value" in subprocess.check_output("zfs get test:property data/dst", shell=True, encoding="utf-8")) == properties ) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def test__run_periodic_snapshot_tasks__recursive(): with patch("zettarepl.zettarepl.create_snapshot") as create_snapshot: with patch("zettarepl.zettarepl.get_empty_snapshots_for_deletion", Mock(return_value=[])): zettarepl = Zettarepl(Mock(), Mock()) zettarepl._run_periodic_snapshot_tasks( datetime(2018, 9, 1, 15, 11), [ Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M"), Mock(dataset="data", recursive=True, naming_schema="snap-%Y-%m-%d_%H-%M"), ] ) create_snapshot.assert_called_once_with(ANY, Snapshot("data", "snap-2018-09-01_15-11"), True, ANY)
def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') osc.die_with_parent() if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl default_level = logging.DEBUG elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl default_level = logging.INFO else: default_level = logging.getLevelName(self.debug_level) setup_logging("", "DEBUG", self.log_handler) oqlh = ObserverQueueLoggingHandler(self.observer_queue) oqlh.setFormatter( logging.Formatter( '[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s', '%Y/%m/%d %H:%M:%S')) logging.getLogger("zettarepl").addHandler(oqlh) for handler in logging.getLogger("zettarepl").handlers: handler.addFilter(LongStringsFilter()) handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level)) c = Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) c.subscribe( 'core.reconfigure_logging', lambda *args, **kwargs: reconfigure_logging('zettarepl_file')) definition = Definition.from_data(self.definition, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10)
def test__replication_tasks_for_periodic_snapshot_tasks(): zettarepl = Zettarepl(Mock(), Mock()) pst1 = Mock() pst2 = Mock() pst3 = Mock() rt1 = Mock(spec=ReplicationTask) rt1.periodic_snapshot_tasks = [pst1, pst2] rt2 = Mock(spec=ReplicationTask) rt2.periodic_snapshot_tasks = [] assert zettarepl._replication_tasks_for_periodic_snapshot_tasks([rt1, rt2], [pst1, pst3]) == [rt1]
def test__transport_for_replication_tasks(): zettarepl = Zettarepl(Mock(), Mock()) t1 = Mock() t2 = Mock() rt1 = Mock(transport=t1) rt2 = Mock(transport=t2) rt3 = Mock(transport=t1) assert sorted(zettarepl._transport_for_replication_tasks([rt1, rt2, rt3]), key=lambda t: [t1, t2].index(t[0])) == [ (t1, [rt1, rt3]), (t2, [rt2]), ]
def test_push_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = Definition.from_data(yaml.load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def test__run_periodic_snapshot_tasks__alphabetical(): with patch("zettarepl.zettarepl.create_snapshot") as create_snapshot: with patch("zettarepl.zettarepl.get_empty_snapshots_for_deletion", Mock(return_value=[])): zettarepl = Zettarepl(Mock(), Mock()) zettarepl._run_periodic_snapshot_tasks( datetime(2018, 9, 1, 15, 11), [ Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M-2d"), Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M-1w"), ] ) assert create_snapshot.call_count == 2 create_snapshot.assert_has_calls([ call(ANY, Snapshot("data", "snap-2018-09-01_15-11-1w"), False, ANY), call(ANY, Snapshot("data", "snap-2018-09-01_15-11-2d"), False, ANY), ])
def create_zettarepl(definition): local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock(return_value=None) zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) return zettarepl
def test_replication_resume(caplog, transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """)) definition["replication-tasks"]["src"]["transport"] = transport definition = Definition.from_data(definition) caplog.set_level(logging.INFO) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert any( "Resuming replication for destination dataset" in record.message for record in caplog.get_records("call") ) assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_does_not_remove_the_last_snapshot_left(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2020-05-07_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2020-05-23_00-00", shell=True) data = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: false naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" lifetime: P30D """)) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2020, 6, 25, 0, 0)) assert list_snapshots(local_shell, "data/src", False) == [Snapshot("data/src", "2020-05-23_00-00")]
def run(args): definition = load_definition(args.definition_path) clock = Clock(args.once) tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() zettarepl = Zettarepl(scheduler, local_shell) zettarepl.set_tasks(definition.tasks) zettarepl.run()
def test_zvol_replication(as_root): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) if as_root: subprocess.check_call("zfs create -V 1M data/src", shell=True) else: subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create -V 1M data/src/zvol", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert len(list_snapshots(local_shell, "data/dst", False)) == 2 if not as_root: assert len(list_snapshots(local_shell, "data/dst/zvol", False)) == 2
def test_source_retention_multiple_sources(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_02-00", shell=True) definition = Definition.from_data(yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: [data/src/a, data/src/b] target-dataset: data/dst recursive: false periodic-snapshot-tasks: - src auto: true retention-policy: source hold-pending-snapshots: true """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst/a", False) == [Snapshot("data/dst/a", "2018-10-01_02-00")] assert list_snapshots(local_shell, "data/dst/b", False) == [Snapshot("data/dst/b", "2018-10-01_02-00")]
def test_push_remote_retention(retention_policy, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True) data = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true """)) data["replication-tasks"]["src"].update(**retention_policy) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst", False) == remains
def test_hold_pending_snapshots(hold_pending_snapshots, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: - id: src dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: - id: src direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: source hold-pending-snapshots: """ + yaml.dump(hold_pending_snapshots) + """ """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/src", False) == remains
def test_hold_pending_snapshots(retention_policy, remains): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True) data = yaml.load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: - id: src direction: pull transport: type: local source-dataset: data/src target-dataset: data/dst naming-schema: "%Y-%m-%d_%H-%M" recursive: true auto: true """)) data["replication-tasks"][0].update(**retention_policy) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0)) assert list_snapshots(local_shell, "data/dst", False) == remains
def run(args): try: definition = Definition.from_data(yaml.load(args.definition_path)) except yaml.YAMLError as e: sys.stderr.write(f"Definition syntax error: {e!s}\n") sys.exit(1) except jsonschema.exceptions.ValidationError as e: sys.stderr.write(f"Definition validation error: {e!s}\n") sys.exit(1) except ValueError as e: sys.stderr.write(f"{e!s}\n") sys.exit(1) clock = Clock(args.once) tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() zettarepl = Zettarepl(scheduler, local_shell) zettarepl.set_tasks(definition.tasks) zettarepl.run()
def test_does_not_remove_the_last_snapshot_left(snapshots__removal_dates__result): snapshots, removal_dates, result = snapshots__removal_dates__result subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/src2", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/child", shell=True) subprocess.check_call("zfs create data/src2", shell=True) for snapshot in snapshots: subprocess.check_call(f"zfs snapshot {snapshot}", shell=True) data = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: false naming-schema: "%Y-%m-%d-%H-%M" schedule: minute: "*" hour: "*" day-of-month: "*" month: "*" day-of-week: "*" lifetime: P30D """)) definition = Definition.from_data(data) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell, use_removal_dates=True) zettarepl.set_tasks(definition.tasks) with patch("zettarepl.zettarepl.get_removal_dates", Mock(return_value=removal_dates)): zettarepl._run_local_retention(datetime(2021, 4, 19, 17, 0)) assert list_snapshots(local_shell, "data/src", False) + list_snapshots(local_shell, "data/src2", False) == [ snapshots[i] for i in result ]
def test_pull_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: - id: src direction: pull transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true naming-schema: - "%Y-%m-%d_%H-%M" auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) assert len(list_snapshots(local_shell, "data/dst", False)) == 2
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') start_daemon_thread(target=watch_parent) if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl debug_level = "DEBUG" elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl debug_level = "INFO" else: debug_level = self.debug_level setup_logging("", debug_level, self.log_handler) for handler in logging.getLogger().handlers: handler.addFilter(LongStringsFilter()) definition = Definition.from_data(self.definition) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client(py_exceptions=True) as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client(py_exceptions=True) as c: c.call("vmware.periodic_snapshot_task_end", context) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": self.zettarepl.set_tasks(Definition.from_data(args).tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id)
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, properties, compression): if transport["type"] != "ssh" and compression: return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst_parent", shell=True) if dst_exists: subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if dst_parent_is_readonly: subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["properties"] = properties if compression: definition["replication-tasks"]["src"]["compression"] = compression definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2 assert (("test-value" in subprocess.check_output( "zfs get test:property data/dst_parent/dst", shell=True, encoding="utf-8")) == properties) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_preserves_clone_origin(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/iocage", shell=True) subprocess.check_call("zfs create data/src/iocage/child", shell=True) subprocess.check_call("zfs create data/src/iocage/child/dataset", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/iocage/child/dataset/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_14-00", shell=True) subprocess.check_call("zfs create data/src/iocage/another", shell=True) subprocess.check_call("zfs create data/src/iocage/another/child", shell=True) subprocess.check_call( "zfs clone data/src/iocage/child/dataset@2019-11-08_14-00 " "data/src/iocage/another/child/clone", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00", shell=True) assert (subprocess.check_output( "zfs get -H origin data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/src/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6 definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true properties: true replicate: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert (subprocess.check_output( "zfs get -H origin data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/dst/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6
def test_parallel_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src-a: dataset: data/src/a recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" src-b: dataset: data/src/b recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src-a: direction: push transport: type: ssh hostname: localhost private-key: | -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8 meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5 pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3 v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8 JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7 ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N -----END RSA PRIVATE KEY----- host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4=" source-dataset: data/src/a target-dataset: data/dst/a recursive: true periodic-snapshot-tasks: - src-a auto: true retention-policy: none speed-limit: 100000 src-b: direction: push transport: type: ssh hostname: localhost private-key: | -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8 meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5 pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3 v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8 JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7 ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N -----END RSA PRIVATE KEY----- host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4=" source-dataset: data/src/b target-dataset: data/dst/b recursive: true periodic-snapshot-tasks: - src-b auto: true retention-policy: none speed-limit: 100000 """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 10 <= end - start <= 15 zettarepl._spawn_retention.assert_called_once() assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1 subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 20 <= end - start <= 25 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_snapshot_gone(transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 2 """)) definition["replication-tasks"]["src"]["transport"] = transport definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) deleted = False def resume_replications_mock(*args, **kwargs): nonlocal deleted if not deleted: # Snapshots are already listed, and now we remove one of them to simulate PULL replication # from remote system that has `allow_empty_snapshots: false`. Only do this once. subprocess.check_call("zfs destroy data/src@2018-10-01_01-00", shell=True) deleted = True return resume_replications(*args, **kwargs) with patch("zettarepl.replication.run.resume_replications", resume_replications_mock): zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, replicate, properties, encrypted, has_encrypted_child, dst_parent_encrypted): if replicate and not properties: return if encrypted and has_encrypted_child: # If parent is encrypted, child is also encrypted return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) create_dataset("data/src", encrypted) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) if has_encrypted_child: create_dataset("data/src/child", True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) create_dataset("data/dst_parent", dst_parent_encrypted) if dst_exists: subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if dst_parent_is_readonly: subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["replicate"] = replicate definition["replication-tasks"]["src"]["properties"] = properties definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) if dst_exists and properties and encrypted and not dst_parent_encrypted: error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskError), error assert error.error == ("Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated " "dataset 'data/dst_parent/dst'") return error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2 assert ( ("test-value" in subprocess.check_output("zfs get test:property data/dst_parent/dst", shell=True, encoding="utf-8")) == properties ) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') osc.die_with_parent() if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl default_level = logging.DEBUG elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl default_level = logging.INFO else: default_level = logging.getLevelName(self.debug_level) setup_logging("", "DEBUG", self.log_handler) oqlh = ObserverQueueLoggingHandler(self.observer_queue) oqlh.setFormatter( logging.Formatter( '[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s', '%Y/%m/%d %H:%M:%S')) logging.getLogger("zettarepl").addHandler(oqlh) for handler in logging.getLogger("zettarepl").handlers: handler.addFilter(LongStringsFilter()) handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level)) c = Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) c.subscribe('core.reconfigure_logging', lambda *args, **kwargs: reconfigure_logging()) definition = Definition.from_data(self.definition, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client() as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id, job=True) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client() as c: c.call("vmware.periodic_snapshot_task_end", context, job=True) except ClientException as e: if e.error: logger.error( "Unhandled exception in ZettareplProcess._observer: %r", e.error) if e.trace: logger.error( "Unhandled exception in ZettareplProcess._observer:\n%s", e.trace["formatted"]) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": definition = Definition.from_data(args, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) self.zettarepl.set_tasks(definition.tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id) self.observer_queue.put( ReplicationTaskError(task_id, "Task not found"))
def test_parallel_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src-a: dataset: data/src/a recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" src-b: dataset: data/src/b recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src-a: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/a target-dataset: data/dst/a recursive: true periodic-snapshot-tasks: - src-a auto: true retention-policy: none speed-limit: 100000 src-b: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/b target-dataset: data/dst/b recursive: true periodic-snapshot-tasks: - src-b auto: true retention-policy: none speed-limit: 100000 """)) set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"]) set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"]) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 10 <= end - start <= 15 zettarepl._spawn_retention.assert_called_once() assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1 subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 20 <= end - start <= 25 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1